From b6963b661958573935839b5d817b001ef56ef24c Mon Sep 17 00:00:00 2001 From: Pedro Amaral Date: Wed, 29 May 2024 17:09:44 -0300 Subject: [PATCH 01/10] Version 1.4.3 This new version of pysal/spreg brings several new features, performance enhancements and bug fixes. The main contributors to this new version are Luc Anselin, Pedro Amaral, Renan Serenini and Lisa Singh. Updates: 1- Introduction of the DGP module a. The DGP module allows for the creation of spatial models for specific Data Generation Processes (DGP) to support simulation exercises. These include the creation of error term vectors (classic and spatial), dependent and independent variables, spatially correlated or not, and other elements for OLS, SAR, SLX, SDM, SARAR models, etc. 2- Introduction of new specification tests a. The Koley-Bera (2024) tests for WX and SDM, and their variants, have been included to the diagnostics suite. b. The Common Factor Hypothesis test has been added to Spatial Durbin Models (GM and ML). 3- Impact estimation a. The estimation of average direct impact (ADI), average indirect impact (AII), and average total impact (ATI) in summary results has been added to models with a spatial lag of the dependent variable. 4- Endogenous Spatial Regimes estimation a. Methods for endogenous spatial regimes estimation based on Anselin and Amaral (2023) have been added, such as OLS_Endog_Regimes and GM_Lag_Endog_Regimes. 5- A flag to allow for the printing of the table with the coefficients' results and their inference straight in LaTeX format 6- Skater_reg now allows for the estimation of Spatial Lag models with a common spatial lag across regimes. A method adapted from Mojena (1977)'s Rule Two has also been introduced to find the optimal number of regimes for the endogenous spatial regimes models. 7- Several minor performance enhancements and bug fixes. --- spreg/__init__.py | 1 + spreg/dgp.py | 835 ++++++++++++++++++++++++++++++++++ spreg/diagnostics_sp.py | 260 ++++++++++- spreg/error_sp.py | 94 ++-- spreg/error_sp_het.py | 116 ++++- spreg/error_sp_het_regimes.py | 42 +- spreg/error_sp_hom.py | 67 ++- spreg/error_sp_hom_regimes.py | 39 +- spreg/error_sp_regimes.py | 15 +- spreg/ml_error.py | 7 +- spreg/ml_error_regimes.py | 2 +- spreg/ml_lag.py | 39 +- spreg/ml_lag_regimes.py | 72 ++- spreg/ols.py | 3 +- spreg/ols_regimes.py | 46 +- spreg/output.py | 233 +++++++--- spreg/robust.py | 4 +- spreg/skater_reg.py | 129 ++++-- spreg/sputils.py | 58 +++ spreg/twosls.py | 1 - spreg/twosls_regimes.py | 2 +- spreg/twosls_sp.py | 126 ++--- spreg/twosls_sp_regimes.py | 127 ++++-- spreg/user_output.py | 29 +- spreg/utils.py | 74 ++- 25 files changed, 2080 insertions(+), 341 deletions(-) mode change 100644 => 100755 spreg/__init__.py create mode 100644 spreg/dgp.py mode change 100644 => 100755 spreg/twosls_sp_regimes.py diff --git a/spreg/__init__.py b/spreg/__init__.py old mode 100644 new mode 100755 index 845e95e8..2de967b5 --- a/spreg/__init__.py +++ b/spreg/__init__.py @@ -2,6 +2,7 @@ import contextlib from importlib.metadata import PackageNotFoundError, version +from .dgp import * from .diagnostics import * from .diagnostics_panel import * from .diagnostics_sp import * diff --git a/spreg/dgp.py b/spreg/dgp.py new file mode 100644 index 00000000..8f4e34c4 --- /dev/null +++ b/spreg/dgp.py @@ -0,0 +1,835 @@ +""" +Generating spatial models for specific DGP + +""" + +__author__ = "Luc Anselin lanselin@gmail.com,\ + Pedro Amaral pedrovma@gmail.com,\ + Renan Serenini renan.serenini@uniroma1.it" + +import numpy as np +import math +import libpysal +from scipy.linalg import expm +import spreg + +__all__ = [ + "make_error", + "make_x", + "make_wx", + "make_xb", + "make_wxg", + "dgp_errproc", + "dgp_ols", + "dgp_slx", + "dgp_sperror", + "dgp_slxerror", + "dgp_lag", + "dgp_spdurbin", + "dgp_lagerr", + "dgp_gns", + "dgp_mess" +] + + +def make_error(rng,n,mu=0,varu=1,method='normal'): + """ + make_error: generate error term for a given distribution + + Arguments: + ---------- + rng: random number object + n: number of observations + mu: mean (when needed) + varu: variance (when needed) + method: type of distribution, one of + normal, laplace, cauchy, lognormal + + Returns: + -------- + u: nx1 vector of random errors + + Examples + -------- + + >>> import numpy as np + >>> from spreg import make_error + >>> rng = np.random.default_rng(12345) + >>> make_error(rng,5) + array([[-1.42382504], + [ 1.26372846], + [-0.87066174], + [-0.25917323], + [-0.07534331]]) + + """ + # normal - standard normal is default + if method == 'normal': + sdu = math.sqrt(varu) + u = rng.normal(loc=mu,scale=sdu,size=n).reshape(n,1) + # laplace with thicker tails + elif method == 'laplace': + sdu = math.sqrt(varu/2.0) + u = rng.laplace(loc=mu,scale=sdu,size=n).reshape(n,1) + # cauchy, ill-behaved, no mean or variance defined + elif method == 'cauchy': + u = rng.standard_cauchy(size=n).reshape(n,1) + elif method == 'lognormal': + sdu = math.sqrt(varu) + u = rng.lognormal(mean=mu,sigma=sdu,size=n).reshape(n,1) + # all other yield warning + else: + print('Warning: Unsupported distribution') + u = None + return u + +def make_x(rng,n,mu=[0],varu=[1],cor=0,method='uniform'): + """ + make_x: generate a matrix of k columns of x for a given distribution + + Arguments: + ---------- + rng: random number object + n: number of observations + mu: mean as a list + varu: variance as a list + cor: correlation as a float (for bivariate normal only) + method: type of distribution, one of + uniform, normal, bivnormal (bivariate normal) + + Returns: + -------- + x: nxk matrix of x variables + + Note: + ----- + Uniform and normal generate separate draws, bivariate normal generates + correlated draws + + Examples + -------- + + >>> import numpy as np + >>> from spreg import make_x + >>> rng = np.random.default_rng(12345) + >>> make_x(rng,5,mu=[0,1],varu=[1,4]) + array([[0.78751508, 2.30580253], + [1.09728308, 4.14520464], + [2.76215497, 1.29373239], + [2.3426149 , 4.6609906 ], + [1.35484323, 6.52500165]]) + + """ + # check on k dimension + k = len(mu) + if k == len(varu): + # initialize + x = np.zeros((n,k)) + for i in range(k): + # uniform - range is derived from variance since var = (1/12)range^2 + # range is found as square root of 12 times variance + # for 0-1, varu should be 0.0833333 + # low is always 0 + if method == 'uniform': + sdu = math.sqrt(12.0*varu[i]) + x[:,i] = rng.uniform(low=0,high=sdu,size=n) + # normal - independent normal draws + elif method == 'normal': + sdu = math.sqrt(varu[i]) + x[:,i] = rng.normal(loc=mu[i],scale=sdu,size=n) + # bivariate normal - only for k=2 + elif method == 'bivnormal': + if k != 2: + print('Error: Wrong dimension for k') + x = None + return x + else: + ucov = cor* math.sqrt(varu[0]*varu[1]) + mcov = [[varu[0],ucov],[ucov,varu[1]]] + x = rng.multivariate_normal(mean=mu,cov=mcov,size=n) + return x + else: + print('Warning: Unsupported distribution') + x = None + else: + x = None + return x + +def make_wx(x,w,o=1): + """ + make_wx: generate a matrix spatially lagged x given matrix x + + x must be previously generated using make_x, no constant included + + Arguments: + ---------- + x: x matrix - no constant + w: row-standardized spatial weights in spreg format + o: order of contiguity, default o=1 + + Returns: + -------- + wx: nx(kxo) matrix of spatially lagged x variables + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_wx + >>> rng = np.random.default_rng(12345) + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> make_wx(x,w)[0:5,:] + array([[1.12509217], + [1.87409079], + [1.36225472], + [2.1491645 ], + [2.80255786]]) + + """ + if w.n != x.shape[0]: + print("Error: incompatible weights dimensions") + return None + w1x = libpysal.weights.lag_spatial(w,x) + wx = w1x + if o > 1: + for i in range(1,o): + whx = libpysal.weights.lag_spatial(w,w1x) + w1x = whx + wx = np.hstack((wx,whx)) + return wx + + +def make_xb(x,beta): + """ + make_xb: generate a column xb as matrix x (constant added) + times list beta (includes coefficient for constant term) + + Arguments: + ---------- + x: n x (k-1) matrix for x variables + beta: k length list of regression coefficients + + Returns: + -------- + xb: nx1 vector of x times beta + + Examples + -------- + + >>> import numpy as np + >>> from spreg import make_x, make_xb + >>> rng = np.random.default_rng(12345) + >>> x = make_x(rng,5,mu=[0,1],varu=[1,4]) + >>> make_xb(x,[1,2,3]) + array([[ 9.49243776], + [15.63018007], + [10.4055071 ], + [19.66820159], + [23.28469141]]) + """ + n = x.shape[0] + k = x.shape[1] + if k+1 != len(beta): + print("Error: Incompatible dimensions") + return None + else: + b = np.array(beta)[:,np.newaxis] + x1=np.hstack((np.ones((n,1)),x)) # include constant + xb = np.dot(x1,b) + return xb + +def make_wxg(wx,gamma): + """ + make_wxg: generate a column wxg as matrix wx (no constant) + times list gamma (coefficient of spatially lagged x) + + Arguments: + ---------- + wx: n x ((k-1)xo) matrix for spatially lagged x variables of all orders + gamma: (k-1)*o length list of regression coefficients for spatially lagged x + + Returns: + -------- + wxg: nx1 vector of wx times gamma + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_wx, make_wxg + >>> rng = np.random.default_rng(12345) + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> wx = make_wx(x,w) + >>> print(wx.shape) + (25, 1) + >>> make_wxg(wx,[2,4])[0:5,:] + array([[ 2.25018434, 4.50036868], + [ 3.74818158, 7.49636316], + [ 2.72450944, 5.44901889], + [ 4.298329 , 8.59665799], + [ 5.60511572, 11.21023145]]) + + """ + k = wx.shape[1] + if (k > 1): + if k != len(gamma): + print("Error: Incompatible dimensions") + return None + else: + g = np.array(gamma)[:,np.newaxis] + wxg = np.dot(wx,g) + else: # gamma is a scalar + wxg = wx * gamma + return wxg + +def dgp_errproc(u,w,lam=0.5,model='sar',imethod='power_exp'): + """ + dgp_sar: generates pure spatial error process + + Arguments: + ---------- + u: random error vector + w: spatial weights object + lam: spatial autoregressive parameter + model: type of process ('sar' or 'ma') + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + -------- + y : vector of observations following a pure SAR process + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, dgp_errproc + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> dgp_errproc(u,w)[0:5,:] + array([[-1.43760658], + [ 0.69778271], + [-0.7750646 ], + [-0.47750452], + [-0.72377417]]) + + """ + n0 = u.shape[0] + if w.n != n0: + print("Error: incompatible weights dimensions") + return None + if model == 'sar': + y = spreg.utils.inverse_prod(w,u,lam,inv_method=imethod) + elif model == 'ma': + y = u + lam * libpysal.weights.lag_spatial(w,u) + else: + print("Error: unsupported model type") + return None + return y + + +def dgp_ols(u,xb): + """ + dgp_ols: generates y for non-spatial process with given xb and error term u + + Arguments: + ---------- + u: random error vector + xb: vector of xb + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, dgp_ols + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> dgp_ols(u,xb)[0:5,:] + array([[5.22803968], + [3.60291127], + [1.02632633], + [1.37589879], + [5.07165754]]) + + """ + n1 = u.shape[0] + n2 = xb.shape[0] + if n1 != n2: + print("Error: dimension mismatch") + return None + y = xb + u + return y + +def dgp_slx(u,xb,wxg): + """ + dgp_slx: generates y for SLX with given xb, wxg, and error term u + + Arguments: + ---------- + u: random error vector + xb: vector of xb + wxg: vector of wxg + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, make_wx, make_wxg, dgp_slx + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> wx = make_wx(x,w) + >>> wxg = make_wxg(wx,[2]) + >>> dgp_slx(u, xb, wxg)[0:5,:] + array([[8.85854389], + [7.17524694], + [3.83674621], + [4.73103929], + [8.37023076]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + n2 = wxg.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + elif n1 != n2: + print("Error: dimension mismatch") + return None + y = xb + wxg + u + return y + +def dgp_sperror(u,xb,w,lam=0.5,model='sar',imethod='power_exp'): + """ + dgp_sperror: generates y for spatial error model with given xb, weights, + spatial parameter lam, error term, method for inverse transform + + Arguments: + ---------- + u: random error + xb: vector of xb + w: spatial weights + lam: spatial coefficient + model: type of process ('sar' or 'ma') + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, dgp_sperror + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> dgp_sperror(u, xb, w)[0:5,:] + array([[5.21425813], + [3.03696553], + [1.12192347], + [1.15756751], + [4.42322667]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + if n0 != n1: + print("Error: incompatible weights dimensions") + return None + elif w.n != n1: + print("Error: incompatible weights dimensions") + return None + if model == 'sar': + u1 = spreg.utils.inverse_prod(w,u,lam,inv_method=imethod) + elif model == 'ma': + u1 = u + lam * libpysal.weights.lag_spatial(w,u) + else: + print("Error: unsupported model type") + return None + y = xb + u1 + return y + +def dgp_slxerror(u,xb,wxg,w,lam=0.5,model='sar',imethod='power_exp'): + """ + dgp_sperror: generates y for SLX spatial error model with xb, wxg, weights, + spatial parameter lam, model type (sar or ma), + error term, method for inverse transform + + Arguments: + ---------- + u: random error + xb: vector of xb + wxg: vector of wxg + w: spatial weights + lam: spatial coefficient + model: type of process ('sar' or 'ma') + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, make_wx, make_wxg, dgp_slxerror + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> wx = make_wx(x,w) + >>> wxg = make_wxg(wx,[2]) + >>> dgp_slxerror(u,xb,wxg,w)[0:5,:] + array([[8.84476235], + [6.6093012 ], + [3.93234334], + [4.51270801], + [7.7217999 ]]) + """ + + n0 = u.shape[0] + n1 = xb.shape[0] + n2 = wxg.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + elif n1 != n2: + print("Error: dimension mismatch") + return None + if w.n != n1: + print("Error: incompatible weights dimensions") + return None + if model == 'sar': + u1 = spreg.utils.inverse_prod(w,u,lam,inv_method=imethod) + elif model == 'ma': + u1 = u + lam * libpysal.weights.lag_spatial(w,u) + else: + print("Error: unsupported model type") + return None + y = xb + wxg + u1 + return y + +def dgp_lag(u,xb,w,rho=0.5,imethod='power_exp'): + """ + dgp_lag: generates y for spatial lag model with xb, weights, + spatial parameter rho, + error term, method for inverse transform + + Arguments: + ---------- + u: random error + xb: vector of xb + w: spatial weights + rho: spatial coefficient + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, dgp_lag + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> dgp_lag(u, xb, w)[0:5,:] + array([[10.16582326], + [ 7.75359864], + [ 5.39821733], + [ 5.62244672], + [ 8.868168 ]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + if w.n != n1: + print("Error: incompatible weights dimensions") + return None + y1 = xb + u + y = spreg.utils.inverse_prod(w,y1,rho,inv_method=imethod) + return y + +def dgp_spdurbin(u,xb,wxg,w,rho=0.5,imethod='power_exp'): + """ + dgp_spdurbin: generates y for spatial Durbin model with xb, wxg, weights, + spatial parameter rho, + error term, method for inverse transform + + Arguments: + ---------- + u: random error + xb: vector of xb + wxg: vector of wxg + w: spatial weights + rho: spatial coefficient + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, make_wx, make_wxg, dgp_spdurbin + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> wx = make_wx(x,w) + >>> wxg = make_wxg(wx,[2]) + >>> dgp_spdurbin(u,xb,wxg,w)[0:5,:] + array([[18.06895353], + [15.18686487], + [11.95080505], + [12.55220513], + [15.75805066]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + n2 = wxg.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + elif n1 != n2: + print("Error: dimension mismatch") + return None + if w.n != n1: + print("Error: incompatible weights dimensions") + y1 = xb + wxg + u + y = spreg.utils.inverse_prod(w,y1,rho,inv_method=imethod) + return y + +def dgp_lagerr(u,xb,w,rho=0.5,lam=0.2,model='sar',imethod='power_exp'): + """ + dgp_lagerr: generates y for spatial lag model with sar or ma errors + with xb, weights, + spatial parameter rho, spatial parameter lambda, + model for spatial process, + error term, method for inverse transform + + Arguments: + ---------- + u: random error + xb: vector of xb + w: spatial weights + rho: spatial coefficient for lag + lam: spatial coefficient for error + model: spatial process for error + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, dgp_lagerr + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> dgp_lagerr(u, xb, w)[0:5,:] + array([[10.13845523], + [ 7.53009531], + [ 5.40644034], + [ 5.51132886], + [ 8.58872366]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + if w.n != n1: + print("Error: incompatible weights dimensions") + return None + if model == 'sar': + u1 = spreg.utils.inverse_prod(w,u,lam,inv_method=imethod) + elif model == 'ma': + u1 = u + lam * libpysal.weights.lag_spatial(w,u) + else: + print("Error: unsupported model type") + return None + y1 = xb + u1 + y = spreg.utils.inverse_prod(w,y1,rho,inv_method=imethod) + return y + +def dgp_gns(u,xb,wxg,w,rho=0.5,lam=0.2,model='sar',imethod='power_exp'): + """ + dgp_gns: generates y for general nested model with sar or ma errors + with xb, wxg, weights, + spatial parameter rho, spatial parameter lambda, + model for spatial process, + error term, method for inverse transform + + Arguments: + ---------- + u: random error + xb: vector of xb + wxg: vector of wxg + w: spatial weights + rho: spatial coefficient for lag + lam: spatial coefficient for error + model: spatial process for error + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, make_wx, make_wxg, dgp_gns + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> wx = make_wx(x,w) + >>> wxg = make_wxg(wx,[2]) + >>> dgp_gns(u,xb,wxg,w)[0:5,:] + array([[18.04158549], + [14.96336153], + [11.95902806], + [12.44108728], + [15.47860632]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + n2 = wxg.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + elif n1 != n2: + print("Error: dimension mismatch") + return None + if w.n != n1: + print("Error: incompatible weights dimensions") + if model == 'sar': + u1 = spreg.utils.inverse_prod(w,u,lam,inv_method=imethod) + elif model == 'ma': + u1 = u + lam * libpysal.weights.lag_spatial(w,u) + else: + print("Error: unsupported model type") + return None + y1 = xb + wxg + u1 + y = spreg.utils.inverse_prod(w,y1,rho,inv_method=imethod) + return y + +def dgp_mess(u,xb,w,rho=0.5): + """ + dgp_mess: generates y for MESS spatial lag model with xb, weights, + spatial parameter rho (gets converted into alpha), + sigma/method for the error term + + Arguments: + ---------- + u: random error + xb: vector of xb + w: spatial weights + rho: spatial coefficient (converted into alpha) + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, dgp_mess + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> dgp_mess(u, xb, w)[0:5,:] + array([[10.12104421], + [ 7.45561055], + [ 5.32807674], + [ 5.55549492], + [ 8.62685145]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + if w.n != n1: + print("Error: incompatible weights dimensions") + return None + bigw = libpysal.weights.full(w)[0] + alpha=np.log(1-rho) #convert between rho and alpha + aw=-alpha*bigw # inverse exponential is -alpha + xbu = xb + u + y = np.dot(expm(aw),xbu) + return y + +def _test(): + import doctest + + start_suppress = np.get_printoptions()["suppress"] + np.set_printoptions(suppress=True) + doctest.testmod() + np.set_printoptions(suppress=start_suppress) + +if __name__ == "__main__": + _test() + \ No newline at end of file diff --git a/spreg/diagnostics_sp.py b/spreg/diagnostics_sp.py index bf0dbfe7..42dadc0d 100644 --- a/spreg/diagnostics_sp.py +++ b/spreg/diagnostics_sp.py @@ -1,7 +1,7 @@ """ Spatial diagnostics module """ -__author__ = "Luc Anselin luc.anselin@asu.edu, Daniel Arribas-Bel darribas@asu.edu" +__author__ = "Luc Anselin lanselin@gmail.com, Daniel Arribas-Bel darribas@asu.edu, Pedro Amaral pedrovma@gmail.com" from .utils import spdot @@ -19,7 +19,7 @@ class LMtests: """ - Lagrange Multiplier tests. Implemented as presented in :cite:`Anselin1996a` + Lagrange Multiplier tests. Implemented as presented in :cite:`Anselin1996a` and :cite:`KoleyBera2024` Attributes ---------- @@ -37,6 +37,11 @@ class LMtests: * 'rlme': Robust LM error test * 'lml' : LM lag test * 'rlml': Robust LM lag test + * 'sarma': LM SARMA test + * 'lmwx': LM test for WX + * 'rlmwx': Robust LM WX test + * 'lmspdurbin': Joint test for SDM + * 'rlmdurlag': Robust LM Lag - SDM Parameters ---------- @@ -54,9 +59,20 @@ class LMtests: (Only if 'rlml' or 'all' was in tests). Pair of statistic and p-value for the Robust LM lag test. sarma : tuple - (Only if 'rlml' or 'all' was in tests). Pair of statistic + (Only if 'sarma' or 'all' was in tests). Pair of statistic and p-value for the SARMA test. - + lmwx : tuple + (Only if 'lmwx' or 'all' was in tests). Pair of statistic + and p-value for the LM test for WX. + rlmwx : tuple + (Only if 'rlmwx' or 'all' was in tests). Pair of statistic + and p-value for the Robust LM WX test. + rlmdurlag : tuple + (Only if 'rlmdurlag' or 'all' was in tests). Pair of statistic + and p-value for the Robust LM Lag - SDM test. + lmspdurbin : tuple + (Only if 'lmspdurbin' or 'all' was in tests). Pair of statistic + and p-value for the Joint test for SDM. Examples -------- @@ -122,15 +138,36 @@ class LMtests: >>> print(round(lms.sarma[0],4), round(lms.sarma[1],4)) 4.1907 0.123 + + LM test for WX: + + >>> print(round(lms.lmwx[0],4), round(lms.lmwx[1],4)) + 1.3377 0.5123 + + Robust LM WX test: + + >>> print(round(lms.rlmwx[0],4), round(lms.rlmwx[1],4)) + 3.4532 0.1779 + + Robust LM Lag - SDM: + >>> print(round(lms.rlmdurlag[0],4), round(lms.rlmdurlag[1],4)) + 3.0971 0.0784 + + Joint test for SDM: + + >>> print(round(lms.lmspdurbin[0],4), round(lms.lmspdurbin[1],4)) + 4.4348 0.2182 """ def __init__(self, ols, w, tests=["all"]): cache = spDcache(ols, w) if tests == ["all"]: - tests = ["lme", "lml", "rlme", "rlml", "sarma"] - if "lme" in tests: + tests = ["lme", "lml", "rlme", "rlml", "sarma", "lmwx", "lmspdurbin", "rlmwx", + "rlmdurlag", "lmslxerr"] # added back in for access + if any(test in ["lme", "lmslxerr"] for test in tests): + #if "lme" in tests: self.lme = lmErr(ols, w, cache) - if "lml" in tests: + if any(test in ["lml", "rlmwx"] for test in tests): self.lml = lmLag(ols, w, cache) if "rlme" in tests: self.rlme = rlmErr(ols, w, cache) @@ -138,7 +175,17 @@ def __init__(self, ols, w, tests=["all"]): self.rlml = rlmLag(ols, w, cache) if "sarma" in tests: self.sarma = lmSarma(ols, w, cache) - + #if any(test in ["lmwx", "rlmdurlag", "lmslxerr"] for test in tests): + if any(test in ["lmwx", "rlmdurlag","lmslxerr"] for test in tests): + self.lmwx = lm_wx(ols, w) + if any(test in ["lmspdurbin", "rlmdurlag", "rlmwx"] for test in tests): + self.lmspdurbin = lm_spdurbin(ols, w) + if "rlmwx" in tests: + self.rlmwx = rlm_wx(ols, self.lmspdurbin, self.lml) + if "rlmdurlag" in tests: + self.rlmdurlag = rlm_durlag(self.lmspdurbin, self.lmwx) + if "lmslxerr" in tests: #currently removed - LA added back in for access + self.lmslxerr = lm_slxerr(ols, self.lme, self.lmwx) class MoranRes: """ @@ -630,6 +677,186 @@ def lmSarma(ols, w, spDcache): pval = chisqprob(lm, 2) return (lm[0][0], pval[0][0]) +def lm_wx(reg, w): + """ + LM test for WX. Implemented as presented in Koley & Bera (2024) :cite:`KoleyBera2024`. + + Attributes + ---------- + reg : OLS + Instance from an OLS regression + w : W + Spatial weights instance + + Returns + ------- + lmwx : tuple + Pair of statistic and p-value for the LM test for WX. + + """ + + # preliminaries + # set up X1 (constant) and X (no constant) as x1 and xx + x1 = reg.x + xx = x1[:,1:] + # WX + wx = w.sparse * xx + # end of preliminaries + # X'W'u + xtwtu = wx.T @ reg.u + # X'W'X1(X1'X1)-1X1WX + mx1 = wx.T @ x1 + mx = (mx1 @ reg.xtxi) @ mx1.T + xwwx = wx.T @ wx + xqx = xwwx - mx + xqxi = la.inv(xqx) + # RSgamma: (X'W'u)'(X'Q1X)-1(X'W'u) / sig2n + xpwpu = wx.T @ reg.u + rsg1 = (xpwpu.T @ xqxi) @ xpwpu + rsgam = rsg1[0][0] / reg.sig2n + pval = chisqprob(rsgam, (reg.k - 1)) + rsgamma = (rsgam,pval) + return(rsgamma) + +def lm_spdurbin(reg,w): + """ + Joint test for SDM. Implemented as presented in Koley & Bera (2024) :cite:`KoleyBera2024`. + + Attributes + ---------- + reg : OLS + Instance from an OLS regression + w : W + Spatial weights instance + + Returns + ------- + lmspdurbin : tuple + Pair of statistic and p-value for the Joint test for SDM. + + """ + + # preliminaries + # set up X1 (constant) and X (no constant) as x1 and xx + x1 = reg.x + xx = x1[:,1:] + k = x1.shape[1] + # WX + wx = w.sparse * xx + # X1b + xb = reg.predy + # WX1b + wxb = w.sparse * xb + # Wy + wy = w.sparse * reg.y + # y'W'e / sig2n + drho = (wy.T @ reg.u) / reg.sig2n + # X'W'e / sign2n + dgam = (wx.T @ reg.u) / reg.sig2n + # P = T = tr(W2 + W'W) + pp = w.trcWtW_WW + # end of preliminaries + # J_11: block matrix with X1'X1 and n/2sig2n + jj1a = np.hstack((reg.xtx,np.zeros((k,1)))) + jj1b = np.hstack((np.zeros((1,k)),np.array([reg.n/(2.0*reg.sig2n)]).reshape(1,1))) + jj11 = np.vstack((jj1a,jj1b)) + # J_12: matrix with k-1 rows X1'WX1b and X1'WX, and 1 row of zeros + jj12a = np.hstack((x1.T @ wxb, x1.T @ wx)) + jj12 = np.vstack((jj12a,np.zeros((1,k)))) + # J_22 matrix with diagonal elements b'X1'W'WX1b + T.sig2n and X'W'WX + # and off-diagonal element b'X1'W'WX + jj22a = wxb.T @ wxb + pp * reg.sig2n + jj22a = jj22a.reshape(1,1) + wxbtwx = (wxb.T @ wx).reshape(1,k-1) + jj22b = np.hstack((jj22a,wxbtwx)) + wxtwx = wx.T @ wx + jj22c = np.hstack((wxbtwx.T,wxtwx)) + jj22 = np.vstack((jj22b,jj22c)) + # J^22 (the inverse) from J^22 = (J_22 - J_21.J_11^-1.J_12)^-1 + jj11i = la.inv(jj11) + j121121 = (jj12.T @ jj11i) @ jj12 + jj22i1 = jj22 - j121121 + jj22i = la.inv(jj22i1) + # rescale by sig2n + jj22i = jj22i * reg.sig2n + # statistic + dd = np.vstack((drho,dgam)) + rsjoint = (dd.T @ jj22i) @ dd + rsjoint = rsjoint[0][0] + pval = chisqprob(rsjoint, k) + rsrhogam = (rsjoint, pval) + return(rsrhogam) + +def rlm_wx(reg,lmspdurbin,lmlag): + """ + Robust LM WX test. Implemented as presented in Koley & Bera (2024) :cite:`KoleyBera2024`. + + Attributes + ---------- + reg : OLS + Instance from an OLS regression + lmspdurbin : tuple + Joint test for SDM as in lm_spdurbin function + lmlag : tuple + LM Lag test as in lmLag function + + Returns + ------- + rlmwx : tuple + Pair of statistic and p-value for the Robust LM WX test. + + """ + # robust gamma = rsjoint - rsrho + rsgams = lmspdurbin[0] - lmlag[0] + pval = chisqprob(rsgams,(reg.k - 1)) + rsgamstar = (rsgams, pval) + return(rsgamstar) + +def rlm_durlag(lmspdurbin,lmwx): + """ + Robust LM Lag - SDM. Implemented as presented in Koley & Bera (2024) :cite:`KoleyBera2024`. + + Attributes + ---------- + lmspdurbin : tuple + Joint test for SDM as in lm_spdurbin function + lmwx : tuple + LM test for WX as in lm_wx function + + Returns + ------- + rlmwx : tuple + Pair of statistic and p-value for the Robust LM Lag - SDM test. + """ + + # robust rho = rsjoint - rsgam + rsrhos = lmspdurbin[0] - lmwx[0] + pval = chisqprob(rsrhos,1) + rsrhostar = (rsrhos, pval) + return(rsrhostar) + +def lm_slxerr(reg,lmerr,lmwx): + """ + Joint test for Error and WX. Implemented as presented in Koley & Bera (2024) :cite:`KoleyBera2024`. + + Attributes + ---------- + reg : OLS + Instance from an OLS regression + lmerr : tuple + LM Error test as in lmErr function + lmwx : tuple + LM test for WX as in lm_wx function + + Returns + ------- + rlmwx : tuple + Pair of statistic and p-value for the Joint test for Error and WX. + """ + rslamgam = lmerr[0] + lmwx[0] + pval = chisqprob(rslamgam,reg.k) + rslamgamma = (rslamgam,pval) + return(rslamgamma) def get_mI(reg, w, spDcache): """ @@ -725,15 +952,20 @@ def akTest(iv, w, spDcache): return (mi, ak[0][0], pval[0][0]) -def comfac_test(lambd, beta, gamma, vm): +def comfac_test(rho, beta, gamma, vm): """ - Computes the Spatial Common Factor Hypothesis test as shown in Anselin (1988, p. 226-229) + Computes the Spatial Common Factor Hypothesis test as shown in Anselin (1988, p. 226-229). + Note that for the Common Factor Hypothesis test to be valid, gamma has to equal + *negative* rho times beta for all beta parameters. + That is, when rho is positive, a positive beta means gamma must be negative and vice versa. + For a negative rho, beta, and gamma must have the same sign. + If those signs are not compatible, the test will not be meaningful. Parameters ---------- - lambd : float - Spatial autoregressive coefficient (as in lambd*Wy) + rho : float + Spatial autoregressive coefficient (as in rho*Wy) beta : array Coefficients of the exogenous (not spatially lagged) variables, without the constant (as in X*beta) gamma : array @@ -751,8 +983,8 @@ def comfac_test(lambd, beta, gamma, vm): with k-1 degrees of freedom """ - g = lambd * beta + gamma - G = np.vstack((lambd * np.eye(beta.shape[0]), np.eye(beta.shape[0]), beta.T)) + g = rho * beta + gamma + G = np.vstack((rho * np.eye(beta.shape[0]), np.eye(beta.shape[0]), beta.T)) GVGi = la.inv(np.dot(G.T, np.dot(vm, G))) W = np.dot(g.T, np.dot(GVGi, g))[0][0] diff --git a/spreg/error_sp.py b/spreg/error_sp.py index 765fddc8..406ac155 100644 --- a/spreg/error_sp.py +++ b/spreg/error_sp.py @@ -2,7 +2,7 @@ Spatial Error Models module """ -__author__ = "Luc Anselin luc.anselin@asu.edu, \ +__author__ = "Luc Anselin lanselin@gmail.com, \ Daniel Arribas-Bel darribas@asu.edu, \ Pedro V. Amaral pedro.amaral@asu.edu" @@ -37,7 +37,9 @@ class BaseGM_Error(RegressionPropsY): independent (exogenous) variable, excluding the constant w : Sparse matrix Spatial weights sparse matrix - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- betas : array @@ -87,7 +89,7 @@ class BaseGM_Error(RegressionPropsY): [ 0.3257]]) """ - def __init__(self, y, x, w): + def __init__(self, y, x, w, hard_bound=False): # 1a. OLS --> \tilde{betas} ols = OLS.BaseOLS(y=y, x=x) @@ -97,7 +99,7 @@ def __init__(self, y, x, w): # 1b. GMM --> \tilde{\lambda1} moments = _momentsGM_Error(w, ols.u) - lambda1 = optim_moments(moments) + lambda1 = optim_moments(moments, hard_bound=hard_bound) # 2a. OLS -->\hat{betas} xs = get_spFilter(w, lambda1, self.x) @@ -147,7 +149,9 @@ class GM_Error(BaseGM_Error): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside maximum/minimum bounds. Attributes ---------- output : dataframe @@ -287,26 +291,29 @@ class GM_Error(BaseGM_Error): """ def __init__( - self, y, x, w, slx_lags=0, vm=False, name_y=None, name_x=None, name_w=None, name_ds=None, latex=False - ): + self, y, x, w, slx_lags=0, vm=False, name_y=None, name_x=None, name_w=None, name_ds=None, latex=False, + hard_bound=False): n = USER.check_arrays(y, x) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # intialize in case of None, contains constant set_warn(self, warn) self.title = "GM SPATIALLY WEIGHTED LEAST SQUARES" if slx_lags >0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant self.title += " WITH SLX (SDEM)" - BaseGM_Error.__init__(self, y=y, x=x_constant, w=w.sparse) + BaseGM_Error.__init__(self, y=y, x=x_constant, w=w.sparse, hard_bound=hard_bound) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # already includes constant self.name_x.append("lambda") self.name_w = USER.set_name_w(name_w, w) self.output = pd.DataFrame(self.name_x, columns=['var_names']) @@ -338,7 +345,9 @@ class BaseGM_Endog_Error(RegressionPropsY): this should not contain any variables from x) w : Sparse matrix Spatial weights sparse matrix - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- betas : array @@ -396,7 +405,7 @@ class BaseGM_Endog_Error(RegressionPropsY): """ - def __init__(self, y, x, yend, q, w): + def __init__(self, y, x, yend, q, w, hard_bound=False): # 1a. TSLS --> \tilde{betas} tsls = TSLS.BaseTSLS(y=y, x=x, yend=yend, q=q) @@ -407,7 +416,7 @@ def __init__(self, y, x, yend, q, w): # 1b. GMM --> \tilde{\lambda1} moments = _momentsGM_Error(w, tsls.u) - lambda1 = optim_moments(moments) + lambda1 = optim_moments(moments, hard_bound=hard_bound) # 2a. 2SLS -->\hat{betas} xs = get_spFilter(w, lambda1, self.x) @@ -468,7 +477,9 @@ class GM_Endog_Error(BaseGM_Endog_Error): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -645,23 +656,27 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize for None, includes constant set_warn(self, warn) self.title = "GM SPATIALLY WEIGHTED TWO STAGE LEAST SQUARES" if slx_lags >0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant self.title += " WITH SLX (SDEM)" - BaseGM_Endog_Error.__init__(self, y=y, x=x_constant, w=w.sparse, yend=yend, q=q) + BaseGM_Endog_Error.__init__(self, y=y, x=x_constant, w=w.sparse, yend=yend, q=q, hard_bound=hard_bound) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # already includes constant self.name_yend = USER.set_name_yend(name_yend, yend) self.name_z = self.name_x + self.name_yend self.name_z.append("lambda") @@ -726,7 +741,9 @@ class GM_Combo(BaseGM_Endog_Error): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -934,12 +951,14 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) set_warn(self, warn) if slx_lags == 0: yend2, q2 = set_endog(y, x_constant[:, 1:], w, yend, q, w_lags, lag_q) @@ -947,7 +966,7 @@ def __init__( yend2, q2, wx = set_endog(y, x_constant[:, 1:], w, yend, q, w_lags, lag_q, slx_lags) x_constant = np.hstack((x_constant, wx)) - BaseGM_Endog_Error.__init__(self, y=y, x=x_constant, w=w.sparse, yend=yend2, q=q2) + BaseGM_Endog_Error.__init__(self, y=y, x=x_constant, w=w.sparse, yend=yend2, q=q2, hard_bound=hard_bound) self.rho = self.betas[-2] self.predy_e, self.e_pred, warn = sp_att( @@ -956,11 +975,13 @@ def __init__( set_warn(self, warn) self.title = "SPATIALLY WEIGHTED 2SLS - GM-COMBO MODEL" if slx_lags > 0: - name_x += USER.set_name_spatial_lags(name_x, slx_lags) +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant self.title += " WITH SLX (GNSM)" self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # constant already in list self.name_yend = USER.set_name_yend(name_yend, yend) self.name_yend.append(USER.set_name_yend_sp(self.name_y)) self.name_z = self.name_x + self.name_yend @@ -1026,6 +1047,9 @@ class GMM_Error(GM_Error, GM_Endog_Error, GM_Combo, GM_Error_Het, GM_Endog_Error Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. **kwargs : keywords Additional arguments to pass on to the estimators. See the specific functions for details on what can be used. @@ -1209,54 +1233,54 @@ class GMM_Error(GM_Error, GM_Endog_Error, GM_Combo, GM_Error_Het, GM_Endog_Error def __init__( self, y, x, w, yend=None, q=None, estimator='het', add_wy=False, slx_lags=0, vm=False, name_y=None, name_x=None, name_w=None, name_yend=None, - name_q=None, name_ds=None, latex=False, **kwargs): + name_q=None, name_ds=None, latex=False, hard_bound=False, **kwargs): if estimator == 'het': if yend is None and not add_wy: GM_Error_Het.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) elif yend is not None and not add_wy: GM_Endog_Error_Het.__init__(self, y=y, x=x, yend=yend, q=q, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) elif add_wy: GM_Combo_Het.__init__(self, y=y, x=x, yend=yend, q=q, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) else: set_warn(self, 'Combination of arguments passed to GMM_Error not allowed. Using default arguments instead.') GM_Error_Het.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound) elif estimator == 'hom': if yend is None and not add_wy: GM_Error_Hom.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) elif yend is not None and not add_wy: GM_Endog_Error_Hom.__init__(self, y=y, x=x, yend=yend, q=q, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) elif add_wy: GM_Combo_Hom.__init__(self, y=y, x=x, yend=yend, q=q, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) else: set_warn(self, 'Combination of arguments passed to GMM_Error not allowed. Using default arguments instead.') GM_Error_Hom.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound) elif estimator == 'kp98': if yend is None and not add_wy: GM_Error.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) elif yend is not None and not add_wy: GM_Endog_Error.__init__(self, y=y, x=x, yend=yend, q=q, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) elif add_wy: GM_Combo.__init__(self, y=y, x=x, yend=yend, q=q, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) else: set_warn(self, 'Combination of arguments passed to GMM_Error not allowed. Using default arguments instead.') GM_Error.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound) else: set_warn(self, 'Combination of arguments passed to GMM_Error not allowed. Using default arguments instead.') GM_Error_Het.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound) def _momentsGM_Error(w, u): diff --git a/spreg/error_sp_het.py b/spreg/error_sp_het.py index 50b838fb..ac2bc598 100755 --- a/spreg/error_sp_het.py +++ b/spreg/error_sp_het.py @@ -2,7 +2,7 @@ Spatial Error with Heteroskedasticity family of models """ -__author__ = "Luc Anselin luc.anselin@asu.edu, \ +__author__ = "Luc Anselin lanselin@gmail.com, \ Pedro V. Amaral pedro.amaral@asu.edu, \ Daniel Arribas-Bel darribas@asu.edu, \ David C. Folch david.folch@asu.edu \ @@ -49,7 +49,9 @@ class BaseGM_Error_Het(RegressionPropsY): an additional stop condition. step1c : boolean If True, then include Step 1c from :cite:`Arraiz2010`. - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- betas : array @@ -107,7 +109,7 @@ class BaseGM_Error_Het(RegressionPropsY): [ 0.4118 0.168 ]] """ - def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, step1c=False): + def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, step1c=False, hard_bound=False): self.step1c = step1c # 1a. OLS --> \tilde{betas} @@ -126,6 +128,13 @@ def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, step1c=False): lambda2 = UTILS.optim_moments(moments, vc1) else: lambda2 = lambda1 + + # Forcing the 1st step lambda to be in the range [-0.9, 0.9] to avoid perfect collinearity in step 2 in case of SLX-Error or GNS models + #if lambda2 > 0.9: + # lambda_old = 0.9 + #elif lambda2 < -0.9: + # lambda_old = -0.9 + #else: lambda_old = lambda2 self.iteration, eps = 0, 1 @@ -147,6 +156,12 @@ def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, step1c=False): self.iteration += 1 self.iter_stop = UTILS.iter_msg(self.iteration, max_iter) + if hard_bound: + if abs(lambda3) >= 0.99: + raise Exception("Spatial error parameter was outside the bounds of -0.99 and 0.99") + else: + if abs(lambda3) >= 0.99: + set_warn(self, "Spatial error parameter was outside the bounds of -0.99 and 0.99") sigma = get_psi_sigma(w, self.u, lambda3) vc3 = get_vc_het(w, wA1, sigma) @@ -197,7 +212,9 @@ class GM_Error_Het(BaseGM_Error_Het): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -354,18 +371,21 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None, includes constant set_warn(self, warn) self.title = "GM SPATIALLY WEIGHTED LEAST SQUARES (HET)" if slx_lags >0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # no constant self.title += " WITH SLX (SDEM)" BaseGM_Error_Het.__init__( self, @@ -375,10 +395,12 @@ def __init__( max_iter=max_iter, step1c=step1c, epsilon=epsilon, + hard_bound = hard_bound ) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # constant already included self.name_x.append("lambda") self.name_w = USER.set_name_w(name_w, w) self.output = pd.DataFrame(self.name_x, columns=['var_names']) @@ -425,7 +447,9 @@ class BaseGM_Endog_Error_Het(RegressionPropsY): If "power_exp", then compute inverse using the power expansion. If "true_inv", then compute the true inverse. Note that true_inv will fail for large n. - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- @@ -510,11 +534,22 @@ def __init__( epsilon=0.00001, step1c=False, inv_method="power_exp", + hard_bound=False, ): self.step1c = step1c # 1a. reg --> \tilde{betas} tsls = TSLS.BaseTSLS(y=y, x=x, yend=yend, q=q) + + if abs(tsls.betas[-1]) <= 0.9: + pass + else: + if tsls.betas[-1] < -0.9: + tsls.betas[-1] = -0.9 + else: + tsls.betas[-1] = 0.9 + tsls.u = tsls.y - spdot(tsls.z, tsls.betas) + self.x, self.z, self.h, self.y = tsls.x, tsls.z, tsls.h, tsls.y self.yend, self.q, self.n, self.k, self.hth = ( tsls.yend, @@ -539,6 +574,13 @@ def __init__( lambda2 = UTILS.optim_moments(moments, vc1) else: lambda2 = lambda1 + + # Forcing the 1st step lambda to be in the range [-0.9, 0.9] to avoid perfect collinearity in step 2 in case of SLX-Error or GNS models + #if lambda2 > 0.9: + # lambda_old = 0.9 + #elif lambda2 < -0.9: + # lambda_old = -0.9 + #else: lambda_old = lambda2 self.iteration, eps = 0, 1 @@ -548,6 +590,16 @@ def __init__( ys = UTILS.get_spFilter(w, lambda_old, self.y) yend_s = UTILS.get_spFilter(w, lambda_old, self.yend) tsls_s = TSLS.BaseTSLS(ys, xs, yend_s, h=self.h) + + if abs(tsls_s.betas[-1]) <= 0.9: + pass + else: + if tsls_s.betas[-1] < -0.9: + tsls_s.betas[-1] = -0.9 + else: + tsls_s.betas[-1] = 0.9 + tsls_s.u = tsls_s.y - spdot(tsls_s.z, tsls_s.betas) + self.predy = spdot(self.z, tsls_s.betas) self.u = self.y - self.predy @@ -563,11 +615,29 @@ def __init__( ) moments_i = UTILS._moments2eqs(wA1, w, self.u) lambda3 = UTILS.optim_moments(moments_i, vc2) + + #if abs(lambda3) <= 0.9: + # pass + #elif lambda3 > 0.9: + # lambda3 = 0.9 + #elif lambda3 < -0.9: + # lambda3 = -0.9 + eps = abs(lambda3 - lambda_old) lambda_old = lambda3 self.iteration += 1 self.iter_stop = UTILS.iter_msg(self.iteration, max_iter) + if hard_bound: + if abs(lambda3) >= 0.99: + raise Exception("Spatial error parameter was outside the bounds of -0.99 and 0.99") + if abs(tsls_s.betas[-1]) >= 0.99: + raise Exception("Spatial lag parameter was outside the bounds of -0.99 and 0.99") + else: + if abs(lambda3) >= 0.99: + set_warn(self, "Spatial error parameter was outside the bounds of -0.99 and 0.99") + if abs(tsls_s.betas[-1]) >= 0.99: + set_warn(self, "Spatial lag parameter was outside the bounds of -0.99 and 0.99") zs = UTILS.get_spFilter(w, lambda3, self.z) P = get_P_hat(self, tsls.hthi, zs) @@ -635,7 +705,9 @@ class GM_Endog_Error_Het(BaseGM_Endog_Error_Het): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -827,18 +899,20 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False ): n = USER.check_arrays(y, x, yend, q) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None, includes constant set_warn(self, warn) self.title = "GM SPATIALLY WEIGHTED TWO STAGE LEAST SQUARES (HET)" if slx_lags > 0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # no constant self.title += " WITH SLX (SDEM)" BaseGM_Endog_Error_Het.__init__( self, @@ -851,10 +925,12 @@ def __init__( step1c=step1c, epsilon=epsilon, inv_method=inv_method, + hard_bound=hard_bound, ) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # constant already included self.name_yend = USER.set_name_yend(name_yend, yend) self.name_z = self.name_x + self.name_yend self.name_z.append("lambda") # listing lambda last @@ -913,7 +989,9 @@ class BaseGM_Combo_Het(BaseGM_Endog_Error_Het): If "power_exp", then compute inverse using the power expansion. If "true_inv", then compute the true inverse. Note that true_inv will fail for large n. - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- @@ -1021,6 +1099,7 @@ def __init__( epsilon=0.00001, step1c=False, inv_method="power_exp", + hard_bound=False, ): BaseGM_Endog_Error_Het.__init__( @@ -1034,6 +1113,7 @@ def __init__( step1c=step1c, epsilon=epsilon, inv_method=inv_method, + hard_bound=hard_bound, ) @@ -1102,7 +1182,9 @@ class GM_Combo_Het(BaseGM_Combo_Het): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -1315,12 +1397,14 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None, includes constant set_warn(self, warn) if slx_lags == 0: @@ -1342,6 +1426,7 @@ def __init__( lag_q=lag_q, epsilon=epsilon, inv_method=inv_method, + hard_bound=hard_bound, ) self.rho = self.betas[-2] self.predy_e, self.e_pred, warn = UTILS.sp_att( @@ -1349,12 +1434,15 @@ def __init__( ) UTILS.set_warn(self, warn) self.title = "SPATIALLY WEIGHTED 2SLS- GM-COMBO MODEL (HET)" + if slx_lags > 0: - name_x += USER.set_name_spatial_lags(name_x, slx_lags) +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # no constant self.title += " WITH SLX (GNSM)" self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # constant already included self.name_yend = USER.set_name_yend(name_yend, yend) self.name_yend.append(USER.set_name_yend_sp(self.name_y)) self.name_z = self.name_x + self.name_yend diff --git a/spreg/error_sp_het_regimes.py b/spreg/error_sp_het_regimes.py index 2f812d74..c5c79100 100644 --- a/spreg/error_sp_het_regimes.py +++ b/spreg/error_sp_het_regimes.py @@ -96,7 +96,9 @@ class GM_Error_Het_Regimes(RegressionPropsY, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -321,6 +323,7 @@ def __init__( name_ds=None, name_regimes=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x) @@ -369,6 +372,7 @@ def __init__( vm, name_x, latex, + hard_bound, ) else: raise Exception( @@ -392,13 +396,13 @@ def __init__( # 1b. GMM --> \tilde{\lambda1} moments = UTILS._moments2eqs(wA1, w.sparse, ols.u) - lambda1 = UTILS.optim_moments(moments) + lambda1 = UTILS.optim_moments(moments, hard_bound=hard_bound) if step1c: # 1c. GMM --> \tilde{\lambda2} sigma = get_psi_sigma(w.sparse, ols.u, lambda1) vc1 = get_vc_het(w.sparse, wA1, sigma) - lambda2 = UTILS.optim_moments(moments, vc1) + lambda2 = UTILS.optim_moments(moments, vc1, hard_bound=hard_bound) else: lambda2 = lambda1 lambda_old = lambda2 @@ -419,7 +423,7 @@ def __init__( sigma_i = get_psi_sigma(w.sparse, self.u, lambda_old) vc_i = get_vc_het(w.sparse, wA1, sigma_i) moments_i = UTILS._moments2eqs(wA1, w.sparse, self.u) - lambda3 = UTILS.optim_moments(moments_i, vc_i) + lambda3 = UTILS.optim_moments(moments_i, vc_i, hard_bound=hard_bound) eps = abs(lambda3 - lambda_old) lambda_old = lambda3 self.iteration += 1 @@ -449,7 +453,7 @@ def __init__( output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) def _error_het_regimes_multi( - self, y, x, regimes, w, slx_lags, cores, max_iter, epsilon, step1c, cols2regi, vm, name_x, latex + self, y, x, regimes, w, slx_lags, cores, max_iter, epsilon, step1c, cols2regi, vm, name_x, latex, hard_bound ): regi_ids = dict( @@ -488,6 +492,7 @@ def _error_het_regimes_multi( self.name_w, self.name_regimes, slx_lags, + hard_bound, ), ) else: @@ -507,6 +512,7 @@ def _error_het_regimes_multi( self.name_w, self.name_regimes, slx_lags, + hard_bound, ) ) @@ -657,7 +663,9 @@ class GM_Endog_Error_Het_Regimes(RegressionPropsY, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -927,6 +935,7 @@ def __init__( summ=True, add_lag=False, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) @@ -984,6 +993,7 @@ def __init__( name_q, add_lag, latex, + hard_bound, ) else: raise Exception( @@ -1024,7 +1034,7 @@ def __init__( # 1b. GMM --> \tilde{\lambda1} moments = UTILS._moments2eqs(wA1, w.sparse, tsls.u) - lambda1 = UTILS.optim_moments(moments) + lambda1 = UTILS.optim_moments(moments, hard_bound=hard_bound) if step1c: # 1c. GMM --> \tilde{\lambda2} @@ -1040,7 +1050,7 @@ def __init__( inv_method, filt=False, ) - lambda2 = UTILS.optim_moments(moments, vc1) + lambda2 = UTILS.optim_moments(moments, vc1, hard_bound=hard_bound) else: lambda2 = lambda1 lambda_old = lambda2 @@ -1077,7 +1087,7 @@ def __init__( inv_method, ) moments_i = UTILS._moments2eqs(wA1, w.sparse, self.u) - lambda3 = UTILS.optim_moments(moments_i, vc2) + lambda3 = UTILS.optim_moments(moments_i, vc2, hard_bound=hard_bound) eps = abs(lambda3 - lambda_old) lambda_old = lambda3 self.iteration += 1 @@ -1135,6 +1145,7 @@ def _endog_error_het_regimes_multi( name_q, add_lag, latex, + hard_bound, ): regi_ids = dict( @@ -1184,6 +1195,7 @@ def _endog_error_het_regimes_multi( self.name_regimes, add_lag, slx_lags, + hard_bound, ), ) else: @@ -1209,6 +1221,7 @@ def _endog_error_het_regimes_multi( self.name_regimes, add_lag, slx_lags, + hard_bound, ) ) @@ -1388,7 +1401,9 @@ class GM_Combo_Het_Regimes(GM_Endog_Error_Het_Regimes): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -1691,6 +1706,7 @@ def __init__( name_ds=None, name_regimes=None, latex=False, + hard_bound=False, ): if regime_lag_sep and not regime_err_sep: set_warn(self, "regime_err_sep set to True when regime_lag_sep=True.") @@ -1773,6 +1789,7 @@ def __init__( summ=False, add_lag=add_lag, latex=latex, + hard_bound=hard_bound, ) if regime_err_sep != True: @@ -1807,12 +1824,13 @@ def _work_error( name_w, name_regimes, slx_lags, + hard_bound, ): w_r, warn = REGI.w_regime(w, regi_ids[r], r, transform=True) y_r = y[regi_ids[r]] x_r = x[regi_ids[r]] model = BaseGM_Error_Het( - y_r, x_r, w_r.sparse, max_iter=max_iter, epsilon=epsilon, step1c=step1c + y_r, x_r, w_r.sparse, max_iter=max_iter, epsilon=epsilon, step1c=step1c, hard_bound=hard_bound, ) set_warn(model, warn) model.w = w_r @@ -1849,6 +1867,7 @@ def _work_endog_error( name_regimes, add_lag, slx_lags, + hard_bound, ): w_r, warn = REGI.w_regime(w, regi_ids[r], r, transform=True) y_r = y[regi_ids[r]] @@ -1872,6 +1891,7 @@ def _work_endog_error( epsilon=epsilon, step1c=step1c, inv_method=inv_method, + hard_bound=hard_bound, ) set_warn(model, warn) if add_lag != False: diff --git a/spreg/error_sp_hom.py b/spreg/error_sp_hom.py index 6208b8af..e51f4ea2 100644 --- a/spreg/error_sp_hom.py +++ b/spreg/error_sp_hom.py @@ -4,7 +4,7 @@ """ -__author__ = "Luc Anselin luc.anselin@asu.edu, Daniel Arribas-Bel darribas@asu.edu" +__author__ = "Luc Anselin lanselin@gmail.com, Daniel Arribas-Bel darribas@asu.edu" from scipy import sparse as SP import numpy as np @@ -50,7 +50,9 @@ class BaseGM_Error_Hom(RegressionPropsY): If A1='hom', then as in :cite:`Anselin2011` (default). If A1='hom_sc' (default), then as in :cite:`Drukker2013` and :cite:`Drukker:2013aa`. - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- betas : array @@ -117,7 +119,7 @@ class BaseGM_Error_Hom(RegressionPropsY): [ -2.40000000e-03 3.00000000e-04 -1.00000000e-04 3.37000000e-02]] """ - def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, A1="hom_sc"): + def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, A1="hom_sc", hard_bound=False): if A1 == "hom": wA1 = get_A1_hom(w) elif A1 == "hom_sc": @@ -133,7 +135,7 @@ def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, A1="hom_sc"): # 1b. GM --> \tilde{\rho} moments = moments_hom(w, wA1, wA2, ols.u) - lambda1 = optim_moments(moments) + lambda1 = optim_moments(moments, hard_bound=hard_bound) lambda_old = lambda1 self.iteration, eps = 0, 1 @@ -148,7 +150,7 @@ def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, A1="hom_sc"): # 2b. GM 2nd iteration --> \hat{\rho} moments = moments_hom(w, wA1, wA2, self.u) psi = get_vc_hom(w, wA1, wA2, self, lambda_old)[0] - lambda2 = optim_moments(moments, psi) + lambda2 = optim_moments(moments, psi, hard_bound=hard_bound) eps = abs(lambda2 - lambda_old) lambda_old = lambda2 self.iteration += 1 @@ -206,7 +208,9 @@ class GM_Error_Hom(BaseGM_Error_Hom): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -357,18 +361,21 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None, includes constant set_warn(self, warn) self.title = "GM SPATIALLY WEIGHTED LEAST SQUARES (HOM)" if slx_lags >0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant self.title += " WITH SLX (SDEM)" BaseGM_Error_Hom.__init__( self, @@ -378,10 +385,12 @@ def __init__( A1=A1, max_iter=max_iter, epsilon=epsilon, + hard_bound=hard_bound ) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # constant already included self.name_x.append("lambda") self.name_w = USER.set_name_w(name_w, w) self.A1 = A1 @@ -429,7 +438,9 @@ class BaseGM_Endog_Error_Hom(RegressionPropsY): al. If A1='hom', then as in :cite:`Anselin2011`. If A1='hom_sc' (default), then as in :cite:`Drukker2013` and :cite:`Drukker:2013aa`. - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- betas : array @@ -505,7 +516,7 @@ class BaseGM_Endog_Error_Hom(RegressionPropsY): """ - def __init__(self, y, x, yend, q, w, max_iter=1, epsilon=0.00001, A1="hom_sc"): + def __init__(self, y, x, yend, q, w, max_iter=1, epsilon=0.00001, A1="hom_sc", hard_bound=False): if A1 == "hom": wA1 = get_A1_hom(w) @@ -529,7 +540,7 @@ def __init__(self, y, x, yend, q, w, max_iter=1, epsilon=0.00001, A1="hom_sc"): # 1b. GM --> \tilde{\rho} moments = moments_hom(w, wA1, wA2, tsls.u) - lambda1 = optim_moments(moments) + lambda1 = optim_moments(moments, hard_bound=hard_bound) lambda_old = lambda1 self.iteration, eps = 0, 1 @@ -545,7 +556,7 @@ def __init__(self, y, x, yend, q, w, max_iter=1, epsilon=0.00001, A1="hom_sc"): # 2b. GM 2nd iteration --> \hat{\rho} moments = moments_hom(w, wA1, wA2, self.u) psi = get_vc_hom(w, wA1, wA2, self, lambda_old, tsls_s.z)[0] - lambda2 = optim_moments(moments, psi) + lambda2 = optim_moments(moments, psi, hard_bound=hard_bound) eps = abs(lambda2 - lambda_old) lambda_old = lambda2 self.iteration += 1 @@ -614,7 +625,9 @@ class GM_Endog_Error_Hom(BaseGM_Endog_Error_Hom): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -806,18 +819,21 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None, includes constant set_warn(self, warn) self.title = "GM SPATIALLY WEIGHTED TWO STAGE LEAST SQUARES (HOM)" if slx_lags > 0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant self.title += " WITH SLX (SDEM)" BaseGM_Endog_Error_Hom.__init__( self, @@ -829,10 +845,12 @@ def __init__( A1=A1, max_iter=max_iter, epsilon=epsilon, + hard_bound=hard_bound, ) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # already includes constant self.name_yend = USER.set_name_yend(name_yend, yend) self.name_z = self.name_x + self.name_yend self.name_z.append("lambda") # listing lambda last @@ -891,7 +909,9 @@ class BaseGM_Combo_Hom(BaseGM_Endog_Error_Hom): al. If A1='hom', then as in :cite:`Anselin2011`. If A1='hom_sc' (default), then as in :cite:`Drukker2013` and :cite:`Drukker:2013aa`. - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- @@ -1003,6 +1023,7 @@ def __init__( max_iter=1, epsilon=0.00001, A1="hom_sc", + hard_bound=False, ): BaseGM_Endog_Error_Hom.__init__( @@ -1015,6 +1036,7 @@ def __init__( A1=A1, max_iter=max_iter, epsilon=epsilon, + hard_bound=hard_bound, ) @@ -1082,7 +1104,9 @@ class GM_Combo_Hom(BaseGM_Combo_Hom): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -1291,12 +1315,14 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None, includes constant set_warn(self, warn) if slx_lags == 0: yend2, q2 = set_endog(y, x_constant[:, 1:], w, yend, q, w_lags, lag_q) @@ -1315,6 +1341,7 @@ def __init__( lag_q=lag_q, max_iter=max_iter, epsilon=epsilon, + hard_bound=hard_bound, ) self.rho = self.betas[-2] self.predy_e, self.e_pred, warn = sp_att( @@ -1323,11 +1350,13 @@ def __init__( set_warn(self, warn) self.title = "SPATIALLY WEIGHTED 2SLS- GM-COMBO MODEL (HOM)" if slx_lags > 0: - name_x += USER.set_name_spatial_lags(name_x, slx_lags) +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant self.title += " WITH SLX (GNSM)" self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # constant already included self.name_yend = USER.set_name_yend(name_yend, yend) self.name_yend.append(USER.set_name_yend_sp(self.name_y)) self.name_z = self.name_x + self.name_yend diff --git a/spreg/error_sp_hom_regimes.py b/spreg/error_sp_hom_regimes.py index f8d7dfae..b4eba259 100644 --- a/spreg/error_sp_hom_regimes.py +++ b/spreg/error_sp_hom_regimes.py @@ -102,7 +102,9 @@ class GM_Error_Hom_Regimes(RegressionPropsY, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -332,6 +334,7 @@ def __init__( name_ds=None, name_regimes=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x) @@ -380,6 +383,7 @@ def __init__( vm, name_x, latex, + hard_bound, ) else: raise Exception( @@ -412,7 +416,7 @@ def __init__( # 1b. GM --> \tilde{\rho} moments = moments_hom(w.sparse, wA1, wA2, ols.u) - lambda1 = optim_moments(moments) + lambda1 = optim_moments(moments, hard_bound=hard_bound) lambda_old = lambda1 self.iteration, eps = 0, 1 @@ -430,7 +434,7 @@ def __init__( # 2b. GM 2nd iteration --> \hat{\rho} moments = moments_hom(w.sparse, wA1, wA2, self.u) psi = get_vc_hom(w.sparse, wA1, wA2, self, lambda_old)[0] - lambda2 = optim_moments(moments, psi) + lambda2 = optim_moments(moments, psi, hard_bound=hard_bound) eps = abs(lambda2 - lambda_old) lambda_old = lambda2 self.iteration += 1 @@ -461,7 +465,7 @@ def __init__( output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) def _error_hom_regimes_multi( - self, y, x, regimes, w, slx_lags, cores, max_iter, epsilon, A1, cols2regi, vm, name_x, latex): + self, y, x, regimes, w, slx_lags, cores, max_iter, epsilon, A1, cols2regi, vm, name_x, latex, hard_bound): regi_ids = dict( (r, list(np.where(np.array(regimes) == r)[0])) for r in self.regimes_set @@ -499,6 +503,7 @@ def _error_hom_regimes_multi( self.name_w, self.name_regimes, slx_lags, + hard_bound, ), ) else: @@ -518,6 +523,7 @@ def _error_hom_regimes_multi( self.name_w, self.name_regimes, slx_lags, + hard_bound, ) ) @@ -665,7 +671,9 @@ class GM_Endog_Error_Hom_Regimes(RegressionPropsY, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -943,6 +951,7 @@ def __init__( summ=True, add_lag=False, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) @@ -999,6 +1008,7 @@ def __init__( name_q, add_lag, latex, + hard_bound, ) else: raise Exception( @@ -1046,7 +1056,7 @@ def __init__( # 1b. GM --> \tilde{\rho} moments = moments_hom(w.sparse, wA1, wA2, tsls.u) - lambda1 = optim_moments(moments) + lambda1 = optim_moments(moments, hard_bound=hard_bound) lambda_old = lambda1 self.iteration, eps = 0, 1 @@ -1073,7 +1083,7 @@ def __init__( # 2b. GM 2nd iteration --> \hat{\rho} moments = moments_hom(w.sparse, wA1, wA2, self.u) psi = get_vc_hom(w.sparse, wA1, wA2, self, lambda_old, tsls_s.z)[0] - lambda2 = optim_moments(moments, psi) + lambda2 = optim_moments(moments, psi, hard_bound=hard_bound) eps = abs(lambda2 - lambda_old) lambda_old = lambda2 self.iteration += 1 @@ -1129,6 +1139,7 @@ def _endog_error_hom_regimes_multi( name_q, add_lag, latex, + hard_bound, ): regi_ids = dict( @@ -1177,6 +1188,7 @@ def _endog_error_hom_regimes_multi( self.name_regimes, add_lag, slx_lags, + hard_bound, ), ) else: @@ -1201,6 +1213,7 @@ def _endog_error_hom_regimes_multi( self.name_regimes, add_lag, slx_lags, + hard_bound, ) ) @@ -1381,7 +1394,9 @@ class GM_Combo_Hom_Regimes(GM_Endog_Error_Hom_Regimes): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -1689,6 +1704,7 @@ def __init__( name_ds=None, name_regimes=None, latex=False, + hard_bound=False, ): if regime_lag_sep and not regime_err_sep: @@ -1771,6 +1787,7 @@ def __init__( summ=False, add_lag=add_lag, latex=latex, + hard_bound=hard_bound, ) if regime_err_sep != True: @@ -1805,12 +1822,13 @@ def _work_error( name_w, name_regimes, slx_lags, + hard_bound, ): w_r, warn = REGI.w_regime(w, regi_ids[r], r, transform=True) y_r = y[regi_ids[r]] x_r = x[regi_ids[r]] model = BaseGM_Error_Hom( - y_r, x_r, w_r.sparse, max_iter=max_iter, epsilon=epsilon, A1=A1 + y_r, x_r, w_r.sparse, max_iter=max_iter, epsilon=epsilon, A1=A1, hard_bound=hard_bound, ) set_warn(model, warn) model.w = w_r @@ -1846,6 +1864,7 @@ def _work_endog_error( name_regimes, add_lag, slx_lags, + hard_bound, ): w_r, warn = REGI.w_regime(w, regi_ids[r], r, transform=True) y_r = y[regi_ids[r]] @@ -1861,7 +1880,7 @@ def _work_endog_error( ) x_constant = USER.check_constant(x_r) model = BaseGM_Endog_Error_Hom( - y_r, x_r, yend_r, q_r, w_r.sparse, max_iter=max_iter, epsilon=epsilon, A1=A1 + y_r, x_r, yend_r, q_r, w_r.sparse, max_iter=max_iter, epsilon=epsilon, A1=A1, hard_bound=hard_bound, ) set_warn(model, warn) if add_lag != False: diff --git a/spreg/error_sp_regimes.py b/spreg/error_sp_regimes.py index 8c62c7d8..7dbe978e 100644 --- a/spreg/error_sp_regimes.py +++ b/spreg/error_sp_regimes.py @@ -80,7 +80,9 @@ class GM_Error_Regimes(RegressionPropsY, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -562,7 +564,9 @@ class GM_Endog_Error_Regimes(RegressionPropsY, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -1200,7 +1204,9 @@ class GM_Combo_Regimes(GM_Endog_Error_Regimes, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -1654,6 +1660,9 @@ class GMM_Error_Regimes(GM_Error_Regimes, GM_Combo_Regimes, GM_Endog_Error_Regim Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. **kwargs : keywords Additional arguments to pass on to the estimators. See the specific functions for details on what can be used. diff --git a/spreg/ml_error.py b/spreg/ml_error.py index b6913d25..0f6017d5 100644 --- a/spreg/ml_error.py +++ b/spreg/ml_error.py @@ -485,13 +485,14 @@ def __init__( y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None includes constant set_warn(self, warn) - self.title = "ML SPATIAL ERROR" if slx_lags >0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant from name_x self.title += " WITH SLX (SDEM)" self.title += " (METHOD = " + method + ")" @@ -501,7 +502,7 @@ def __init__( ) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x self.name_x.append("lambda") self.name_w = USER.set_name_w(name_w, w) self.aic = DIAG.akaike(reg=self) diff --git a/spreg/ml_error_regimes.py b/spreg/ml_error_regimes.py index 721fd115..0b315af4 100644 --- a/spreg/ml_error_regimes.py +++ b/spreg/ml_error_regimes.py @@ -83,7 +83,7 @@ class ML_Error_Regimes(BaseML_Error, REGI.Regimes_Frame): name_regimes : string Name of regimes variable for use in output latex : boolean - Specifies if summary is to be printed in latex format + Specifies if the table with the coefficients' results and their inference is to be printed in LaTeX format Attributes ---------- diff --git a/spreg/ml_lag.py b/spreg/ml_lag.py index c3ec794c..2ebc3d45 100755 --- a/spreg/ml_lag.py +++ b/spreg/ml_lag.py @@ -2,7 +2,8 @@ ML Estimation of Spatial Lag Model """ -__author__ = "Luc Anselin luc.anselin@asu.edu, \ +__author__ = "Luc Anselin lanselin@gmail.com, \ + Pedro V. Amaral pedrovma@gmail.com, \ Serge Rey srey@asu.edu, \ Levi Wolf levi.john.wolf@gmail.com" @@ -11,11 +12,11 @@ from scipy import sparse as sp from scipy.sparse.linalg import splu as SuperLU from .utils import RegressionPropsY, RegressionPropsVM, inverse_prod, set_warn, get_lags -from .sputils import spdot, spfill_diagonal, spinv, spbroadcast +from .sputils import spdot, spfill_diagonal, spinv, _spmultiplier from . import diagnostics as DIAG from . import user_output as USER import pandas as pd -from .output import output, _nonspat_top, _spat_pseudo_r2 +from .output import output, _nonspat_top, _spat_diag_out, _spat_pseudo_r2, _summary_impacts from .w_utils import symmetrize from libpysal import weights @@ -337,6 +338,11 @@ class ML_Lag(BaseML_Lag): if 'ord', Ord eigenvalue method epsilon : float tolerance criterion in mimimize_scalar function and inverse_product + spat_diag : boolean + If True, then compute Common Factor Hypothesis test when applicable + spat_impacts : boolean + If True, include average direct impact (ADI), average indirect impact (AII), + and average total impact (ATI) in summary results vm : boolean if True, include variance-covariance matrix in summary results @@ -394,6 +400,9 @@ class ML_Lag(BaseML_Lag): Akaike information criterion schwarz : float Schwarz criterion + cfh_test : tuple + Common Factor Hypothesis test; tuple contains the pair (statistic, + p-value). Only when it applies (see specific documentation). predy_e : array predicted values from reduced form e_pred : array @@ -585,7 +594,9 @@ def __init__( slx_lags=0, method="full", epsilon=0.0000001, + spat_impacts=True, vm=False, + spat_diag=False, name_y=None, name_x=None, name_w=None, @@ -596,6 +607,7 @@ def __init__( y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # needs to be initialized for none, now with constant set_warn(self, warn) method = method.upper() BaseML_Lag.__init__( @@ -605,24 +617,37 @@ def __init__( self.k += 1 if slx_lags>0: - name_x += USER.set_name_spatial_lags(name_x, slx_lags) + kx = len(name_x) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant self.title = "MAXIMUM LIKELIHOOD SPATIAL LAG WITH SLX - SPATIAL DURBIN MODEL" + " (METHOD = " + method + ")" + var_types = ['x'] * kx + ['wx'] * (kx-1) * slx_lags + ['rho'] else: self.title = "MAXIMUM LIKELIHOOD SPATIAL LAG" + " (METHOD = " + method + ")" + var_types = ['x'] * len(name_x) + ['rho'] + self.slx_lags = slx_lags self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x) + self.name_x = name_x # already has constant name_ylag = USER.set_name_yend_sp(self.name_y) self.name_x.append(name_ylag) # rho changed to last position self.name_w = USER.set_name_w(name_w, w) self.aic = DIAG.akaike(reg=self) self.schwarz = DIAG.schwarz(reg=self) self.output = pd.DataFrame(self.name_x, columns=['var_names']) - self.output['var_type'] = ['x'] * (len(self.name_x)-1) + ['rho'] + self.output['var_type'] = var_types self.output['regime'], self.output['equation'] = (0, 0) self.other_top = _spat_pseudo_r2(self) self.other_top += _nonspat_top(self, ml=True) - output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) + diag_out = None + if spat_diag and slx_lags==1: + diag_out = _spat_diag_out(self, w, 'yend', ml=True) + if spat_impacts and slx_lags == 0: + impacts = _summary_impacts(self, _spmultiplier(w, self.rho)) + try: + diag_out += impacts + except TypeError: + diag_out = impacts + output(reg=self, vm=vm, robust=False, other_end=diag_out, latex=latex) def lag_c_loglik(rho, n, e0, e1, W): # concentrated log-lik for lag model, no constants, brute force diff --git a/spreg/ml_lag_regimes.py b/spreg/ml_lag_regimes.py index a05c00b2..bfed43f1 100644 --- a/spreg/ml_lag_regimes.py +++ b/spreg/ml_lag_regimes.py @@ -11,9 +11,9 @@ import multiprocessing as mp from .ml_lag import BaseML_Lag from .utils import set_warn, get_lags -from platform import system +from .sputils import _spmultiplier import pandas as pd -from .output import output, _nonspat_top, _spat_pseudo_r2 +from .output import output, _nonspat_top, _spat_diag_out, _spat_pseudo_r2, _summary_impacts __all__ = ["ML_Lag_Regimes"] @@ -64,6 +64,11 @@ class ML_Lag_Regimes(BaseML_Lag, REGI.Regimes_Frame): If True, the spatial parameter for spatial lag is also computed according to different regimes. If False (default), the spatial parameter is fixed accross regimes. + spat_diag : boolean + If True, then compute Common Factor Hypothesis test when applicable + spat_impacts : boolean + If True, include average direct impact (ADI), average indirect impact (AII), + and average total impact (ATI) in summary results cores : boolean Specifies if multiprocessing is to be used Default: no multiprocessing, cores = False @@ -120,6 +125,9 @@ class ML_Lag_Regimes(BaseML_Lag, REGI.Regimes_Frame): if 'full': brute force (full matrix computations) if 'ord', Ord eigenvalue method if 'LU', LU sparse matrix decomposition + cfh_test : tuple + Common Factor Hypothesis test; tuple contains the pair (statistic, + p-value). Only when it applies (see specific documentation). epsilon : float tolerance criterion used in minimize_scalar function and inverse_product mean_y : float @@ -309,6 +317,8 @@ def __init__( slx_lags=0, regime_lag_sep=False, cores=False, + spat_diag=True, + spat_impacts=True, vm=False, name_y=None, name_x=None, @@ -328,10 +338,11 @@ def __init__( set_warn(self, warn) name_x = USER.set_name_x(name_x, x_constant, constant=True) - if slx_lags >0: + if slx_lags > 0: lag_x = get_lags(w, x_constant, slx_lags) x_constant = np.hstack((x_constant, lag_x)) name_x += USER.set_name_spatial_lags(name_x, slx_lags) + kwx = lag_x.shape[1] self.name_x_r = USER.set_name_x(name_x, x_constant) + [USER.set_name_yend_sp(name_y)] self.method = method @@ -384,6 +395,8 @@ def __init__( cols2regi=cols2regi, method=method, epsilon=epsilon, + spat_diag=spat_diag, + spat_impacts=spat_impacts, vm=vm, name_y=name_y, name_x=name_x, @@ -407,23 +420,43 @@ def __init__( self.name_x.append("_Global_" + USER.set_name_yend_sp(name_y)) BaseML_Lag.__init__(self, y=y, x=x, w=w, method=method, epsilon=epsilon) self.kf += 1 # Adding a fixed k to account for spatial lag in Chow - # adding a fixed k to account for spatial lag in aic, sc + # adding a k to account for spatial lag in aic, sc self.k += 1 self.chow = REGI.Chow(self) self.aic = DIAG.akaike(reg=self) self.schwarz = DIAG.schwarz(reg=self) self.regime_lag_sep = regime_lag_sep self.output = pd.DataFrame(self.name_x, columns=['var_names']) + self.output['regime'] = x_rlist + ['_Global'] self.output['var_type'] = ['x'] * (len(self.name_x) - 1) + ['rho'] - self.output['regime'] = x_rlist+['_Global'] self.output['equation'] = 0 + self.slx_lags = slx_lags + diag_out = None + if slx_lags > 0: + self.title = ("MAXIMUM LIKELIHOOD SPATIAL DURBIN - REGIMES"+ " (METHOD = "+ method+ ")") + fixed_wx = cols2regi[-(kwx+1):-1].count(False) + kwx = kwx - fixed_wx + if kwx > 0: + for m in self.regimes_set: + r_output = self.output[(self.output['regime'] == str(m)) & (self.output['var_type'] == 'x')] + wx_index = r_output.index[-kwx:] + self.output.loc[wx_index, 'var_type'] = 'wx' + if fixed_wx > 0: + f_wx_index = self.output.index[-(fixed_wx+1):-1] + self.output.loc[f_wx_index, 'var_type'] = 'wx' + if spat_diag and slx_lags == 1: + diag_out = _spat_diag_out(self, w, 'yend', ml=True) + else: + self.title = ("MAXIMUM LIKELIHOOD SPATIAL LAG - REGIMES"+ " (METHOD = "+ method+ ")") + if spat_impacts and slx_lags == 0: + impacts = _summary_impacts(self, _spmultiplier(w, self.rho)) + try: + diag_out += impacts + except TypeError: + diag_out = impacts self.other_top = _spat_pseudo_r2(self) self.other_top += _nonspat_top(self, ml=True) - if slx_lags == 0: - self.title = ("MAXIMUM LIKELIHOOD SPATIAL LAG - REGIMES"+ " (METHOD = "+ method+ ")") - else: - self.title = ("MAXIMUM LIKELIHOOD SPATIAL DURBIN - REGIMES"+ " (METHOD = "+ method+ ")") - output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) + output(reg=self, vm=vm, robust=False, other_end=diag_out, latex=latex) def ML_Lag_Regimes_Multi( self, @@ -437,6 +470,8 @@ def ML_Lag_Regimes_Multi( cols2regi, method, epsilon, + spat_diag, + spat_impacts, vm, name_y, name_x, @@ -558,9 +593,19 @@ def ML_Lag_Regimes_Multi( results[r].other_top = _spat_pseudo_r2(results[r]) results[r].other_top += _nonspat_top(results[r], ml=True) results[r].other_mid = "" - self.output = pd.concat([self.output, pd.DataFrame({'var_names': results[r].name_x, - 'var_type': ['x'] * (len(results[r].name_x) - 1) + ['rho'], - 'regime': r, 'equation': r})], ignore_index=True) + if slx_lags > 0: + kx = (len(results[r].name_x) - 1) // (slx_lags + 1) + var_types = ['x'] * (kx + 1) + ['wx'] * kx * slx_lags + ['rho'] + else: + var_types = ['x'] * (len(results[r].name_x) - 1) + ['rho'] + results[r].output = pd.DataFrame({'var_names': results[r].name_x, + 'var_type': var_types, + 'regime': r, 'equation': r}) + self.output = pd.concat([self.output, results[r].output], ignore_index=True) + if spat_diag and slx_lags == 1: + results[r].other_mid += _spat_diag_out(results[r], None, 'yend', ml=True) + if spat_impacts and slx_lags == 0: + results[r].other_mid += _summary_impacts(results[r], _spmultiplier(results[r].w, results[r].rho)) counter += 1 self.multi = results self.chow = REGI.Chow(self) @@ -597,6 +642,7 @@ def _work( model.k += 1 # add 1 for proper df and aic, sc model.aic = DIAG.akaike(reg=model) model.schwarz = DIAG.schwarz(reg=model) + model.slx_lags = slx_lags return model diff --git a/spreg/ols.py b/spreg/ols.py index f22bac19..8efb9c70 100644 --- a/spreg/ols.py +++ b/spreg/ols.py @@ -175,7 +175,7 @@ class OLS(BaseOLS): name_ds : string Name of dataset for use in output latex : boolean - Specifies if summary is to be printed in latex format + Specifies if the table with the coefficients' results and their inference is to be printed in LaTeX format Attributes ---------- @@ -479,6 +479,7 @@ def __init__( ) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) + self.slx_lags = slx_lags self.title = "ORDINARY LEAST SQUARES" if slx_lags > 0: self.title += " WITH SPATIALLY LAGGED X (SLX)" diff --git a/spreg/ols_regimes.py b/spreg/ols_regimes.py index 20b2acea..66ba3d8e 100755 --- a/spreg/ols_regimes.py +++ b/spreg/ols_regimes.py @@ -9,11 +9,11 @@ import pandas as pd from . import regimes as REGI from . import user_output as USER -from .utils import set_warn, RegressionProps_basic, spdot, RegressionPropsY, get_lags +from .utils import set_warn, RegressionProps_basic, spdot, RegressionPropsY, get_lags, optim_k from .ols import BaseOLS from .robust import hac_multi from .output import output, _spat_diag_out, _nonspat_mid, _nonspat_top - +from .skater_reg import Skater_reg class OLS_Regimes(BaseOLS, REGI.Regimes_Frame, RegressionPropsY): """ @@ -447,6 +447,7 @@ def __init__( else: USER.check_weights(w, y, w_required=False) set_warn(self, warn) + self.slx_lags = slx_lags self.name_x_r = USER.set_name_x(name_x, x_constant) self.constant_regi = constant_regi self.cols2regi = cols2regi @@ -529,7 +530,7 @@ def __init__( self.other_mid += _nonspat_mid(self, white_test=white_test) self.other_top += _nonspat_top(self) if spat_diag: - other_end += _spat_diag_out(self, w, 'ols', moran=moran) + other_end += _spat_diag_out(self, w, 'ols', moran=moran) #Must decide what to do with W. output(reg=self, vm=vm, robust=robust, other_end=other_end, latex=latex) def _ols_regimes_multi( @@ -666,7 +667,7 @@ def _ols_regimes_multi( other_end = "" if spat_diag: self._get_spat_diag_props(x_constant, sig2n_k) - other_end += _spat_diag_out(self, w, 'ols', moran=moran) + #other_end += _spat_diag_out(self, w, 'ols', moran=moran) Need to consider W before implementing output(reg=self, vm=vm, robust=robust, other_end=other_end, latex=latex) def _get_spat_diag_props(self, x, sig2n_k): @@ -706,6 +707,43 @@ def _work( return model +class OLS_Endog_Regimes(OLS_Regimes): + def __init__( + self, y, x, w, n_clusters=None, quorum=-np.inf, trace=True, **kwargs): + + n = USER.check_arrays(y, x) + y = USER.check_y(y, n) + USER.check_weights(w, y, w_required=True) + + # Standardize the variables + x_std = (x - np.mean(x, axis=0)) / np.std(x, axis=0) + + if not n_clusters: + if quorum < 0: + quorum = np.max([(x.shape[1]+1)*10, 30]) + n_clusters_opt = x.shape[0]*0.70//quorum + if n_clusters_opt < 2: + raise ValueError( + "The combination of the values of `N` and `quorum` is not compatible with regimes estimation.") + sk_reg_results = Skater_reg().fit(n_clusters_opt, w, x_std, {'reg':BaseOLS,'y':y,'x':x}, quorum=quorum, trace=True) + n_clusters = optim_k([sk_reg_results._trace[i][1][2] for i in range(1, len(sk_reg_results._trace))]) + self.clusters = sk_reg_results._trace[n_clusters-1][0] + else: + try: + # Call the Skater_reg method based on OLS + sk_reg_results = Skater_reg().fit(n_clusters, w, x_std, {'reg':BaseOLS,'y':y,'x':x}, quorum=quorum, trace=trace) + self.clusters = sk_reg_results.current_labels_ + except Exception as e: + if str(e) == "one or more input arrays have more columns than rows": + raise ValueError("One or more input ended up with more variables than observations. Please check your setting for `quorum`.") + else: + print("An error occurred:", e) + + self._trace = sk_reg_results._trace + self.SSR = [self._trace[i][1][2] for i in range(1, len(self._trace))] + + OLS_Regimes.__init__(self, y, x, regimes=self.clusters, w=w, name_regimes='Skater_reg', **kwargs) + def _test(): import doctest diff --git a/spreg/output.py b/spreg/output.py index 25f94e66..389fd12e 100644 --- a/spreg/output.py +++ b/spreg/output.py @@ -5,6 +5,7 @@ import textwrap as TW import numpy as np import pandas as pd +import math from . import diagnostics as diagnostics from . import diagnostics_tsls as diagnostics_tsls from . import diagnostics_sp as diagnostics_sp @@ -16,8 +17,6 @@ ############################################################################### def output(reg, vm, other_end=False, robust=False, latex=False): - if latex: - print("Warning: Latex output not implemented yet. Using standard output instead.") strSummary = output_start(reg) for eq in reg.output['equation'].unique(): try: @@ -26,7 +25,7 @@ def output(reg, vm, other_end=False, robust=False, latex=False): except: eq = None strSummary, reg = out_part_top(strSummary, reg, eq) - strSummary, reg = out_part_middle(strSummary, reg, robust, m=eq) + strSummary, reg = out_part_middle(strSummary, reg, robust, m=eq, latex=latex) strSummary, reg = out_part_end(strSummary, reg, vm, other_end, m=eq) reg.summary = strSummary reg.output.sort_values(by=['equation', 'regime'], inplace=True) @@ -102,7 +101,7 @@ def out_part_top(strSummary, reg, m): return (strSummary, _reg) -def out_part_middle(strSummary, reg, robust, m=None): +def out_part_middle(strSummary, reg, robust, m=None, latex=False): # Middle part of summary output. # m = None for single models, m = 1,2,3... for multiple equation models if m==None: @@ -127,27 +126,40 @@ def out_part_middle(strSummary, reg, robust, m=None): elif robust == "ogmm": strSummary += "Optimal GMM used to estimate the coefficients and the variance-covariance matrix\n" strSummary += "------------------------------------------------------------------------------------\n" - strSummary += ( - " Variable Coefficient Std.Error %1s-Statistic Probability\n" - % (_reg.__summary["summary_zt"]) - ) - strSummary += "------------------------------------------------------------------------------------\n" + m_output = reg.output[reg.output['equation'] == m] - for row in m_output.iloc[np.lexsort((m_output.index, m_output['regime']))].itertuples(): - try: - strSummary += "%20s %12.5f %12.5f %12.5f %12.5f\n" % ( - row.var_names, - row.coefficients, - row.std_err, - row.zt_stat, - row.prob - ) - except TypeError: # special case for models that do not have inference on the lambda term - strSummary += "%20s %12.5f \n" % ( - row.var_names, - row.coefficients - ) - strSummary += "------------------------------------------------------------------------------------\n" + if latex: + df_1 = m_output.iloc[np.lexsort((m_output.index, m_output['regime']))] + df_2 = df_1.loc[:, ['var_names', 'coefficients', 'std_err', 'zt_stat', 'prob']] + df_2 = df_2.set_axis(['Variable', 'Coefficient', 'Std.Error', _reg.__summary['summary_zt']+'-Statistic', 'Prob.'], axis='columns', copy=False) + cols = df_2.columns.difference(['Variable']) + df_2[cols] = df_2[cols].astype(float).map(lambda x: "%12.5f" % x) + df_2['Variable'] = df_2['Variable'].str.replace("_", "\_").str.replace("%", "\%") + df_inlatex = df_2.style.hide(axis='index').to_latex(hrules=True) + strSummary += df_inlatex + strSummary += "------------------------------------------------------------------------------------\n" + else: + strSummary += ( + " Variable Coefficient Std.Error %1s-Statistic Probability\n" + % (_reg.__summary["summary_zt"]) + ) + strSummary += "------------------------------------------------------------------------------------\n" + + for row in m_output.iloc[np.lexsort((m_output.index, m_output['regime']))].itertuples(): + try: + strSummary += "%20s %12.5f %12.5f %12.5f %12.5f\n" % ( + row.var_names, + row.coefficients, + row.std_err, + row.zt_stat, + row.prob + ) + except TypeError: # special case for models that do not have inference on the lambda term + strSummary += "%20s %12.5f \n" % ( + row.var_names, + row.coefficients + ) + strSummary += "------------------------------------------------------------------------------------\n" try: # Adding info on instruments if they are present name_q = _reg.name_q @@ -212,7 +224,7 @@ def _summary_chow(reg): name_x_r = reg.name_x_r joint, regi = reg.chow.joint, reg.chow.regi sum_text += "\n VARIABLE DF VALUE PROB\n" - if reg.cols2regi == "all": + if reg.cols2regi == "all" or set(reg.cols2regi) == {True}: names_chow = name_x_r[1:] else: names_chow = [name_x_r[1:][i] for i in np.where(reg.cols2regi)[0]] @@ -231,7 +243,7 @@ def _summary_chow(reg): 'df': reg.kr * (reg.nr - 1), 'value': joint[0], 'prob': joint[1]}])], ignore_index=True) for row in reg.output_chow.itertuples(): - sum_text += "%25s %2d %12.3f %9.4f\n" % ( + sum_text += "%20s %2d %12.3f %9.4f\n" % ( row.var_names, row.df, row.value, @@ -241,73 +253,152 @@ def _summary_chow(reg): return sum_text -def _spat_diag_out(reg, w, type, moran=False): +def _spat_diag_out(reg, w, type, moran=False, ml=False): strSummary = "\nDIAGNOSTICS FOR SPATIAL DEPENDENCE\n" - if not moran: - strSummary += ( - "TEST DF VALUE PROB\n" - ) - else: - strSummary += ( - "TEST MI/DF VALUE PROB\n" - ) - cache = diagnostics_sp.spDcache(reg, w) if type == "yend": - mi, ak, ak_p = diagnostics_sp.akTest(reg, w, cache) - reg.ak_test = ak, ak_p - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( - "Anselin-Kelejian Test", - 1, - reg.ak_test[0], - reg.ak_test[1], - ) + strSummary += ( + "TEST DF VALUE PROB\n") + if not ml: + mi, ak, ak_p = diagnostics_sp.akTest(reg, w, cache) + reg.ak_test = ak, ak_p + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "Anselin-Kelejian Test", + 1, + reg.ak_test[0], + reg.ak_test[1], + ) + if any(reg.output['var_type'] == 'rho'): + if reg.slx_lags == 1 and not any(reg.output['var_type'] == 'yend'): + wx_indices = reg.output[(reg.output['var_type'] == 'wx') & (reg.output['regime'] != '_Global')].index + x_indices = [] + for m in reg.output['regime'].unique(): + x_indices.extend(reg.output[(reg.output['regime'] == m) & (reg.output['var_type'] == 'x')].index[1:]) + vm_indices = x_indices + wx_indices.tolist() + reg.output[reg.output['var_type'] == 'rho'].index.tolist() + cft, cft_p = diagnostics_sp.comfac_test(reg.rho, + reg.betas[x_indices], + reg.betas[wx_indices], + reg.vm[vm_indices, :][:, vm_indices]) + reg.cfh_test = cft, cft_p + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "Common Factor Hypothesis Test", + len(wx_indices), + reg.cfh_test[0], + reg.cfh_test[1], + ) + elif type == "ols": - lm_tests = diagnostics_sp.LMtests(reg, w) + strSummary += "- SARMA -\n" + if not moran: + strSummary += ( + "TEST DF VALUE PROB\n" + ) + else: + strSummary += ( + "TEST MI/DF VALUE PROB\n" + ) + lm_tests = diagnostics_sp.LMtests(reg, w, tests=["lme", "lml", "rlme", "rlml", "sarma"]) + if reg.slx_lags == 0: + try: + lm_tests2 = diagnostics_sp.LMtests(reg, w, tests=["lmwx", "lmspdurbin", "rlmdurlag", "rlmwx","lmslxerr"]) + reg.lm_wx = lm_tests2.lmwx + reg.lm_spdurbin = lm_tests2.lmspdurbin + reg.rlm_wx = lm_tests2.rlmwx + reg.rlm_durlag = lm_tests2.rlmdurlag + reg.lm_slxerr = lm_tests2.lmslxerr #currently removed. - LA reinstated + koley_bera = True + except: + koley_bera = False reg.lm_error = lm_tests.lme reg.lm_lag = lm_tests.lml reg.rlm_error = lm_tests.rlme reg.rlm_lag = lm_tests.rlml reg.lm_sarma = lm_tests.sarma + + if moran: moran_res = diagnostics_sp.MoranRes(reg, w, z=True) reg.moran_res = moran_res.I, moran_res.zI, moran_res.p_norm - strSummary += "%-27s %8.4f %9.3f %9.4f\n" % ( + strSummary += "%-27s %8.4f %9.3f %9.4f\n" % ( "Moran's I (error)", reg.moran_res[0], reg.moran_res[1], reg.moran_res[2], ) - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Lagrange Multiplier (lag)", 1, reg.lm_lag[0], reg.lm_lag[1], ) - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Robust LM (lag)", 1, reg.rlm_lag[0], reg.rlm_lag[1], ) - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Lagrange Multiplier (error)", 1, reg.lm_error[0], reg.lm_error[1], ) - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Robust LM (error)", 1, reg.rlm_error[0], reg.rlm_error[1], ) - strSummary += "%-27s %2d %12.3f %9.4f\n\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Lagrange Multiplier (SARMA)", 2, reg.lm_sarma[0], reg.lm_sarma[1], ) + if reg.slx_lags == 0 and koley_bera: + strSummary += ( + "\n- Spatial Durbin -\nTEST DF VALUE PROB\n" + ) + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "LM test for WX", + reg.k-1, + reg.lm_wx[0], + reg.lm_wx[1], + ) + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "Robust LM WX test", + reg.k-1, + reg.rlm_wx[0], + reg.rlm_wx[1], + ) + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "Lagrange Multiplier (lag)", + 1, + reg.lm_lag[0], + reg.lm_lag[1], + ) + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "Robust LM Lag - SDM", + 1, + reg.rlm_durlag[0], + reg.rlm_durlag[1], + ) + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "Joint test for SDM", + reg.k, + reg.lm_spdurbin[0], + reg.lm_spdurbin[1], + ) + #strSummary += ( + # "\n- Spatial Error and WX -\nTEST DF VALUE PROB\n" + #) + #strSummary += "%-27s %2d %12.3f %9.4f\n\n" % ( + # "Joint test for Error and WX", + # reg.k, + # reg.lm_slxerr[0], + # reg.lm_slxerr[1], + #) + return strSummary def _nonspat_top(reg, ml=False): @@ -376,10 +467,10 @@ def _nonspat_mid(reg, white_test=False): strSummary = "\nREGRESSION DIAGNOSTICS\n" if reg.mulColli: - strSummary += "MULTICOLLINEARITY CONDITION NUMBER %16.3f\n\n" % (reg.mulColli) + strSummary += "MULTICOLLINEARITY CONDITION NUMBER %15.3f\n\n" % (reg.mulColli) strSummary += "TEST ON NORMALITY OF ERRORS\n" strSummary += "TEST DF VALUE PROB\n" - strSummary += "%-27s %2d %14.3f %9.4f\n\n" % ( + strSummary += "%-27s %2d %14.3f %9.4f\n\n" % ( "Jarque-Bera", reg.jarque_bera["df"], reg.jarque_bera["jb"], @@ -388,13 +479,13 @@ def _nonspat_mid(reg, white_test=False): strSummary += "DIAGNOSTICS FOR HETEROSKEDASTICITY\n" strSummary += "RANDOM COEFFICIENTS\n" strSummary += "TEST DF VALUE PROB\n" - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Breusch-Pagan test", reg.breusch_pagan["df"], reg.breusch_pagan["bp"], reg.breusch_pagan["pvalue"], ) - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Koenker-Bassett test", reg.koenker_bassett["df"], reg.koenker_bassett["kb"], @@ -409,7 +500,7 @@ def _nonspat_mid(reg, white_test=False): strSummary += ( "TEST DF VALUE PROB\n" ) - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "White", reg.white["df"], reg.white["wh"], @@ -476,3 +567,33 @@ def _summary_iteration(reg): pass return txt + +def _summary_impacts(reg, spmult): + """ + Spatial direct, indirect and total effects in spatial lag model. + Uses multipliers computed by sputils._spmultipliers. + + Attributes + ---------- + reg: spreg regression object + spmult: spatial multipliers as a dictionary + + Returns + ------- + strings with direct, indirect and total effects + + """ + variables = reg.output[reg.output['var_type'].isin(['x', 'wx', 'yend']) & (reg.output.index != 0)] + variables_index = variables.index + m1 = spmult['ati'] + btot = m1 * reg.betas[variables_index] + m2 = spmult['adi'] + bdir = m2 * reg.betas[variables_index] + m3 = spmult['aii'] + bind = m3 * reg.betas[variables_index] + strSummary = "\nSPATIAL LAG MODEL IMPACTS\n" + strSummary += " Variable Direct Indirect Total\n" + for i in range(len(variables)): + strSummary += "%20s %12.4f %12.4f %12.4f\n" % ( + variables['var_names'][variables_index[i]], bdir[i][0], bind[i][0], btot[i][0]) + return strSummary \ No newline at end of file diff --git a/spreg/robust.py b/spreg/robust.py index 7d20b103..caf58910 100644 --- a/spreg/robust.py +++ b/spreg/robust.py @@ -1,5 +1,5 @@ -__author__ = "Luc Anselin luc.anselin@asu.edu, \ - Pedro V. Amaral pedro.amaral@asu.edu, \ +__author__ = "Luc Anselin lanselin@gmail.com, \ + Pedro V. Amaral pedrovma@gmail.com, \ David C. Folch david.folch@asu.edu" import numpy as np diff --git a/spreg/skater_reg.py b/spreg/skater_reg.py index 9bfbd3fd..2b203fbf 100755 --- a/spreg/skater_reg.py +++ b/spreg/skater_reg.py @@ -4,11 +4,11 @@ from scipy.sparse import csgraph as cg from scipy.optimize import OptimizeWarning -from scipy.spatial.distance import cdist from collections import namedtuple from warnings import warn from libpysal.weights import w_subset from .utils import set_endog +from .twosls_regimes import TSLS_Regimes import time import numpy as np import copy @@ -68,7 +68,7 @@ class Skater_reg(object): y : array n*1, dependent variable X : array - n*k, independent variable, exlcuding the constant + n*k, independent variable, not including the constant bw : scalar bandwidth value consisting of either a distance or N nearest neighbors; user specified or obtained using @@ -308,7 +308,9 @@ def fit( print("cut made {}...".format(best_deletion)) if best_deletion.score > prev_score: raise ValueError( - "The score increased with the number of clusters. Please check your data." + ("The score increased with the number of clusters. " + "Please check your data.\nquorum: {}; n_clusters: {}" + ).format(quorum, n_clusters) ) prev_score = best_deletion.score MSF = new_MSF @@ -358,7 +360,7 @@ def score_spreg( Any region below quorum makes the score inf. current_labels: (N,) flat vector of labels expressing the classification of each observation into a region not considering the cut under evaluation. - current_tree: integer indicating the tree label is currently being considered for division + current_tree: integer indicating the tree's label currently being considered for division """ labels, subtree_quorums = self._prep_score( @@ -374,44 +376,62 @@ def score_spreg( if k not in ["reg", "y", "x", "w", "x_nd"] } trees_scores = {} - l_arrays = np.array(all_labels) - w_regi_i = None - for l in set_labels: - x = data_reg["x"][all_labels == l] - if np.linalg.matrix_rank(x) < x.shape[1]: - x = np.delete( - x, np.nonzero(np.abs(np.diag(np.linalg.qr(x)[1])) < 1e-10), 1 - ) - if "w" not in data_reg: - try: - x = np.hstack((np.ones((x.shape[0], 1)), x)) - reg = data_reg["reg"]( - y=data_reg["y"][all_labels == l], x=x, **kargs - ) - except np.linalg.LinAlgError: - x = np.delete(x, np.nonzero(np.ptp(x, axis=0) == 0), 1) - x = np.hstack((np.ones((x.shape[0], 1)), x)) - reg = data_reg["reg"]( - y=data_reg["y"][all_labels == l], x=x, **kargs - ) - - else: - regi_ids = list(np.where(l_arrays == l)[0]) - w_ids = list(map(data_reg["w"].id_order.__getitem__, regi_ids)) - w_regi_i = w_subset(data_reg["w"], w_ids, silence_warnings=True) - try: - x = np.hstack((np.ones((x.shape[0], 1)), x)) - reg = data_reg["reg"]( - y=data_reg["y"][all_labels == l], x=x, w=w_regi_i, **kargs - ) - except np.linalg.LinAlgError: - x = np.delete(x, np.nonzero(np.ptp(x, axis=0) == 0), 1) - reg = data_reg["reg"]( - y=data_reg["y"][all_labels == l], x=x, w=w_regi_i, **kargs - ) - trees_scores[l] = np.sum(reg.u ** 2) - score = sum(trees_scores.values()) + if data_reg["reg"].__name__ == "GM_Lag" or data_reg["reg"].__name__ == "BaseGM_Lag": + try: + x = np.hstack((np.ones((data_reg["x"].shape[0], 1)), data_reg["x"])) + reg = TSLS_Regimes( + y=data_reg["y"], + x=x, + yend=data_reg["yend"], + q=data_reg["q"], + regimes=all_labels,) + except: + x = _const_x(data_reg["x"]) + reg = TSLS_Regimes( + y=data_reg["y"], + x=x, + yend=data_reg["yend"], + q=data_reg["q"], + regimes=all_labels,) + score = np.dot(reg.u.T, reg.u)[0][0] + else: + + for l in set_labels: + x = data_reg["x"][all_labels == l] + if np.linalg.matrix_rank(x) < x.shape[1]: + small_diag_indices = np.abs(np.diag(np.linalg.qr(x)[1])) < 1e-10 + x = x[:, ~small_diag_indices] + + if "w" not in data_reg: + try: + x = np.hstack((np.ones((x.shape[0], 1)), x)) + reg = data_reg["reg"]( + y=data_reg["y"][all_labels == l], x=x, **kargs + ) + except np.linalg.LinAlgError: + x = _const_x(x) + reg = data_reg["reg"]( + y=data_reg["y"][all_labels == l], x=x, **kargs + ) + else: + l_arrays = np.array(all_labels) + + regi_ids = list(np.where(l_arrays == l)[0]) + w_ids = list(map(data_reg["w"].id_order.__getitem__, regi_ids)) + w_regi_i = w_subset(data_reg["w"], w_ids, silence_warnings=True) + try: + x = np.hstack((np.ones((x.shape[0], 1)), x)) + reg = data_reg["reg"]( + y=data_reg["y"][all_labels == l], x=x, w=w_regi_i, **kargs + ) + except np.linalg.LinAlgError: + x = _const_x(x) + reg = data_reg["reg"]( + y=data_reg["y"][all_labels == l], x=x, w=w_regi_i, **kargs + ) + trees_scores[l] = np.dot(reg.u.T, reg.u)[0][0] + score = sum(trees_scores.values()) else: part_scores, score, trees_scores = self._data_reg_none( data, all_labels, l, set_labels @@ -465,14 +485,11 @@ def score_stats( if k not in ["reg", "y", "x", "w", "x_nd"] } trees_scores = {} - l_arrays = np.array(all_labels) - w_regi_i = None for l in set_labels: x = data_reg["x"][all_labels == l] if np.linalg.matrix_rank(x) < x.shape[1]: - x = np.delete( - x, np.nonzero(np.abs(np.diag(np.linalg.qr(x)[1])) < 1e-10), 1 - ) + small_diag_indices = np.abs(np.diag(np.linalg.qr(x)[1])) < 1e-10 + x = x[:, ~small_diag_indices] try: x = np.hstack((np.ones((x.shape[0], 1)), x)) @@ -480,8 +497,7 @@ def score_stats( data_reg["y"][all_labels == l], x, **kargs ).fit() except np.linalg.LinAlgError: - x = np.delete(x, np.nonzero(np.ptp(x, axis=0) == 0), 1) - x = np.hstack((np.ones((x.shape[0], 1)), x)) + x = _const_x(x) reg = data_reg["reg"]( data_reg["y"][all_labels == l], x, **kargs ).fit() @@ -527,6 +543,12 @@ def _data_reg_none(self, data, all_labels, l, set_labels): trees_scores = {l: part_scores[i] for i, l in enumerate(set_labels)} return part_scores, score, trees_scores + def _prep_lag(self, data_reg): + # if the model is a spatial lag, add the lagged dependent variable to the model + data_reg['yend'], data_reg['q'] = set_endog(data_reg["y"], data_reg["x"][:, 1:], data_reg["w"], yend=None, + q=None, w_lags=1, lag_q=True) + return data_reg + def find_cut( self, MSF, @@ -594,6 +616,13 @@ def tqdm(noop, desc=""): zero_in = (labels is not None) and (target_label is not None) best_deletion = deletion(np.nan, np.nan, np.inf) best_d_score = -np.inf + + try: + if data_reg["reg"].__name__ == "GM_Lag" or data_reg["reg"].__name__ == "BaseGM_Lag": + data_reg = self._prep_lag(data_reg) + except: + pass + try: old_score = sum(trees_scores.values()) except: @@ -662,6 +691,6 @@ def tqdm(noop, desc=""): def _const_x(x): - x = np.delete(x, np.nonzero(np.ptp(x, axis=0) == 0), 1) + x = x[:, np.ptp(x, axis=0) != 0] x = np.hstack((np.ones((x.shape[0], 1)), x)) - return x + return x \ No newline at end of file diff --git a/spreg/sputils.py b/spreg/sputils.py index 350033fe..65cae18c 100644 --- a/spreg/sputils.py +++ b/spreg/sputils.py @@ -272,6 +272,64 @@ def spisfinite(a): return np.isfinite(a.sum()) +def _spmultiplier(w, rho, method="simple", mtol=0.00000001): + """" + Spatial Lag Multiplier Calculation + Follows Kim, Phipps and Anselin (2003) (simple), and LeSage and Pace (2009) (full, power) + + Attributes + ---------- + w : PySAL format spatial weights matrix + rho : spatial autoregressive coefficient + method : one of "simple" (default), full" or "power" + mtol : tolerance for power iteration (default=0.00000001) + + Returns + ------- + multipliers : dictionary with + ati = average total impact multiplier + adi = average direct impact multiplier + aii = average indirect impact multiplier + pow = powers used in power approximation (otherwise 0) + + """ + multipliers = {"ati": 1.0, "adi": 1.0, "aii": 1.0} + multipliers["pow"] = 0 + multipliers["ati"] = 1.0 / (1.0 - rho) + n = w.n + if method == "simple": + pass + elif method == "full": + wf = w.full()[0] + id0 = np.identity(n) + irw0 = (id0 - rho * wf) + invirw0 = np.linalg.inv(irw0) + adii0 = np.sum(np.diag(invirw0)) + multipliers["adi"] = adii0 / n + elif method == "power": + wf = w.full()[0] + ws3 = SP.csr_array(wf) + rhop = rho + ww = ws3 + pow = 1 + adi = 1.0 + adidiff = 100.00 + + while adidiff > mtol: + pow = pow + 1 + ww = SP.csr_matrix.dot(ww, ws3) + trw = ww.diagonal().sum() + rhop = rhop * rho + adidiff = rhop * trw / n + adi = adi + adidiff + multipliers["adi"] = adi + multipliers["pow"] = pow + else: + print("Method not supported") + multipliers["aii"] = multipliers["ati"] - multipliers["adi"] + return (multipliers) + + def _test(): import doctest diff --git a/spreg/twosls.py b/spreg/twosls.py index 0fe355b1..c0a5bc39 100644 --- a/spreg/twosls.py +++ b/spreg/twosls.py @@ -159,7 +159,6 @@ def __init__( hthi = la.inv(hth) zth = spdot(z.T, h) hty = spdot(h.T, y) - factor_1 = np.dot(zth, hthi) factor_2 = np.dot(factor_1, zth.T) # this one needs to be in cache to be used in AK diff --git a/spreg/twosls_regimes.py b/spreg/twosls_regimes.py index 50a36853..26458331 100644 --- a/spreg/twosls_regimes.py +++ b/spreg/twosls_regimes.py @@ -353,7 +353,7 @@ def __init__( vm=False, constant_regi="many", cols2regi="all", - regime_err_sep=True, + regime_err_sep=False, name_y=None, name_x=None, cores=False, diff --git a/spreg/twosls_sp.py b/spreg/twosls_sp.py index e5ff8552..a58139bc 100755 --- a/spreg/twosls_sp.py +++ b/spreg/twosls_sp.py @@ -8,14 +8,14 @@ from . import twosls as TSLS from . import user_output as USER from .utils import set_endog, sp_att, set_warn +from .sputils import _spmultiplier import pandas as pd -from .output import output, _spat_diag_out, _spat_pseudo_r2 +from .output import output, _spat_diag_out, _spat_pseudo_r2, _summary_impacts __all__ = ["GM_Lag"] class BaseGM_Lag(TSLS.BaseTSLS): - """ Spatial two stage least squares (S2SLS) (note: no consistency checks, diagnostics or constant added); Anselin (1988) [Anselin1988]_ @@ -169,18 +169,18 @@ class BaseGM_Lag(TSLS.BaseTSLS): """ def __init__( - self, - y, - x, - yend=None, - q=None, - w=None, - w_lags=1, - slx_lags=0, - lag_q=True, - robust=None, - gwk=None, - sig2n_k=False, + self, + y, + x, + yend=None, + q=None, + w=None, + w_lags=1, + slx_lags=0, + lag_q=True, + robust=None, + gwk=None, + sig2n_k=False, ): if slx_lags > 0: @@ -195,7 +195,6 @@ def __init__( class GM_Lag(BaseGM_Lag): - """ Spatial two stage least squares (S2SLS) with results and diagnostics; Anselin (1988) :cite:`Anselin1988` @@ -238,7 +237,10 @@ class GM_Lag(BaseGM_Lag): sig2n_k : boolean If True, then use n-k to estimate sigma^2. If False, use n. spat_diag : boolean - If True, then compute Anselin-Kelejian test + If True, then compute Anselin-Kelejian test and Common Factor Hypothesis test (if applicable) + spat_impacts : boolean + If True, include average direct impact (ADI), average indirect impact (AII), + and average total impact (ATI) in summary results vm : boolean If True, include variance-covariance matrix in summary results @@ -258,7 +260,9 @@ class GM_Lag(BaseGM_Lag): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the bounds of -1 and 1. Attributes ---------- output : dataframe @@ -323,6 +327,9 @@ class GM_Lag(BaseGM_Lag): ak_test : tuple Anselin-Kelejian test; tuple contains the pair (statistic, p-value) + cfh_test : tuple + Common Factor Hypothesis test; tuple contains the pair (statistic, + p-value). Only when it applies (see specific documentation). name_y : string Name of dependent variable for use in output name_x : list of strings @@ -485,28 +492,30 @@ class GM_Lag(BaseGM_Lag): """ def __init__( - self, - y, - x, - yend=None, - q=None, - w=None, - w_lags=1, - lag_q=True, - slx_lags=0, - robust=None, - gwk=None, - sig2n_k=False, - spat_diag=False, - vm=False, - name_y=None, - name_x=None, - name_yend=None, - name_q=None, - name_w=None, - name_gwk=None, - name_ds=None, - latex=False, + self, + y, + x, + yend=None, + q=None, + w=None, + w_lags=1, + lag_q=True, + slx_lags=0, + robust=None, + gwk=None, + sig2n_k=False, + spat_diag=True, + spat_impacts=True, + vm=False, + name_y=None, + name_x=None, + name_yend=None, + name_q=None, + name_w=None, + name_gwk=None, + name_ds=None, + latex=False, + hard_bound=False, ): n = USER.check_arrays(x, yend, q) @@ -514,15 +523,16 @@ def __init__( USER.check_weights(w, y, w_required=True) USER.check_robust(robust, gwk) if robust == "hac" and spat_diag: - set_warn( - self, - "Spatial diagnostics are not available for HAC estimation. The spatial diagnostics have been disabled for this model.", - ) - spat_diag = False + set_warn( + self, + "Spatial diagnostics are not available for HAC estimation. The spatial diagnostics have been disabled for this model.", + ) + spat_diag = False x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # need to check for None and set defaults if slx_lags > 0: - name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant set_warn(self, warn) BaseGM_Lag.__init__( @@ -541,7 +551,7 @@ def __init__( ) self.rho = self.betas[-1] self.predy_e, self.e_pred, warn = sp_att( - w, self.y, self.predy, self.yend[:, -1].reshape(self.n, 1), self.rho + w, self.y, self.predy, self.yend[:, -1].reshape(self.n, 1), self.rho, hard_bound=hard_bound ) set_warn(self, warn) self.title = "SPATIAL TWO STAGE LEAST SQUARES" @@ -549,35 +559,43 @@ def __init__( self.title += " WITH SLX (SPATIAL DURBIN MODEL)" self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) # name_x contains SLX terms for slx_lags > 0 + # self.name_x = USER.set_name_x(name_x, x_constant) # name_x contains SLX terms for slx_lags > 0 + self.name_x = name_x # already contains constant in new setup self.name_yend = USER.set_name_yend(name_yend, yend) self.name_yend.append(USER.set_name_yend_sp(self.name_y)) self.name_z = self.name_x + self.name_yend self.name_q = USER.set_name_q(name_q, q) if slx_lags > 0: # need to remove all but last SLX variables from name_x self.name_x0 = [] - self.name_x0.append(self.name_x[0]) # constant -# print(f"x0 first {self.name_x0}") - kx = int((self.k -self.kstar -1)/(slx_lags +1) ) # number of original exogenous vars + self.name_x0.append(self.name_x[0]) # constant + kx = int((self.k - self.kstar - 1) / (slx_lags + 1)) # number of original exogenous vars self.name_x0.extend(self.name_x[-kx:]) -# print(f"in here {self.name_x0}") self.name_q.extend(USER.set_name_q_sp(self.name_x0, w_lags, self.name_q, lag_q)) + var_types = ['x'] * (kx + 1) + ['wx'] * kx * slx_lags + ['yend'] * (len(self.name_yend) - 1) + ['rho'] else: self.name_q.extend(USER.set_name_q_sp(self.name_x, w_lags, self.name_q, lag_q)) + var_types = ['x'] * len(self.name_x) + ['yend'] * (len(self.name_yend) - 1) + ['rho'] self.name_h = USER.set_name_h(self.name_x, self.name_q) self.robust = USER.set_robust(robust) self.name_w = USER.set_name_w(name_w, w) self.name_gwk = USER.set_name_w(name_gwk, gwk) + self.slx_lags = slx_lags self.output = pd.DataFrame(self.name_x + self.name_yend, columns=['var_names']) - self.output['var_type'] = ['x'] * len(self.name_x) + ['yend'] * (len(self.name_yend)-1) + ['rho'] + self.output['var_type'] = var_types self.output['regime'], self.output['equation'] = (0, 0) self.other_top = _spat_pseudo_r2(self) + diag_out = None if spat_diag: diag_out = _spat_diag_out(self, w, 'yend') - else: - diag_out = None + if spat_impacts and slx_lags == 0: + impacts = _summary_impacts(self, _spmultiplier(w, self.rho)) + try: + diag_out += impacts + except TypeError: + diag_out = impacts output(reg=self, vm=vm, robust=robust, other_end=diag_out, latex=latex) + def _test(): import doctest diff --git a/spreg/twosls_sp_regimes.py b/spreg/twosls_sp_regimes.py old mode 100644 new mode 100755 index db0b23c3..185a0878 --- a/spreg/twosls_sp_regimes.py +++ b/spreg/twosls_sp_regimes.py @@ -5,16 +5,18 @@ __author__ = "Luc Anselin luc.anselin@asu.edu, Pedro V. Amaral pedro.amaral@asu.edu, David C. Folch david.folch@asu.edu" import numpy as np +import pandas as pd +import multiprocessing as mp from . import regimes as REGI from . import user_output as USER -import multiprocessing as mp from .twosls_regimes import TSLS_Regimes, _optimal_weight from .twosls import BaseTSLS -from .utils import set_endog, set_endog_sparse, sp_att, set_warn, sphstack, spdot +from .utils import set_endog, set_endog_sparse, sp_att, set_warn, sphstack, spdot, optim_k from .robust import hac_multi -import pandas as pd -from .output import output, _spat_diag_out, _spat_pseudo_r2 - +from .sputils import _spmultiplier +from .output import output, _spat_diag_out, _spat_pseudo_r2, _summary_impacts +from .skater_reg import Skater_reg +from .twosls_sp import BaseGM_Lag class GM_Lag_Regimes(TSLS_Regimes, REGI.Regimes_Frame): @@ -86,8 +88,11 @@ class GM_Lag_Regimes(TSLS_Regimes, REGI.Regimes_Frame): matrix must have ones along the main diagonal. sig2n_k : boolean If True, then use n-k to estimate sigma^2. If False, use n. + spat_impacts : boolean + If True, include average direct impact (ADI), average indirect impact (AII), + and average total impact (ATI) in summary results spat_diag : boolean - If True, then compute Anselin-Kelejian test + If True, then compute Anselin-Kelejian test and Common Factor Hypothesis test (if applicable) vm : boolean If True, include variance-covariance matrix in summary results @@ -206,6 +211,9 @@ class GM_Lag_Regimes(TSLS_Regimes, REGI.Regimes_Frame): p-value) Only available in dictionary 'multi' when multiple regressions (see 'multi' below for details) + cfh_test : tuple + Common Factor Hypothesis test; tuple contains the pair (statistic, + p-value). Only when it applies (see specific documentation). name_y : string Name of dependent variable for use in output name_x : list of strings @@ -443,14 +451,15 @@ def __init__( w_lags=1, slx_lags=0, lag_q=True, - robust=None, + robust='white', gwk=None, sig2n_k=False, - spat_diag=False, + spat_diag=True, + spat_impacts=True, constant_regi="many", cols2regi="all", - regime_lag_sep=True, - regime_err_sep=True, + regime_lag_sep=False, + regime_err_sep=False, cores=False, vm=False, name_y=None, @@ -462,6 +471,7 @@ def __init__( name_gwk=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x) @@ -501,14 +511,19 @@ def __init__( x_constant = np.hstack((x_constant, wx)) name_slx = USER.set_name_spatial_lags(name_x, slx_lags) name_q.extend(USER.set_name_q_sp(name_slx[-len(name_x):], w_lags, name_q, lag_q, force_all=True)) - name_x += name_slx + name_x += name_slx + if cols2regi == 'all': + cols2regi = REGI.check_cols2regi( + constant_regi, cols2regi, x_constant, yend=yend2, add_cons=False)[0:-1] + else: + cols2regi = REGI.check_cols2regi( + constant_regi, cols2regi, x_constant, yend=yend2, add_cons=False) else: name_q.extend(USER.set_name_q_sp(name_x, w_lags, name_q, lag_q, force_all=True)) yend2, q2 = yend, q + cols2regi = REGI.check_cols2regi( + constant_regi, cols2regi, x_constant, yend=yend2, add_cons=False) self.n = x_constant.shape[0] - cols2regi = REGI.check_cols2regi( - constant_regi, cols2regi, x_constant, yend=yend2, add_cons=False - ) self.cols2regi = cols2regi self.regimes_set = REGI._get_regimes_set(regimes) self.regimes = regimes @@ -522,6 +537,7 @@ def __init__( regime_lag_sep = False self.regime_err_sep = regime_err_sep self.regime_lag_sep = regime_lag_sep + if regime_lag_sep == True: cols2regi += [True] w_i, regi_ids, warn = REGI.w_regimes( @@ -533,7 +549,6 @@ def __init__( min_n=len(cols2regi) + 1, ) set_warn(self, warn) - else: cols2regi += [False] @@ -559,6 +574,7 @@ def __init__( gwk=gwk, sig2n_k=sig2n_k, cols2regi=cols2regi, + spat_impacts=spat_impacts, spat_diag=spat_diag, vm=vm, name_y=name_y, @@ -570,6 +586,7 @@ def __init__( name_gwk=name_gwk, name_ds=name_ds, latex=latex, + hard_bound=hard_bound, ) else: if regime_lag_sep == True: @@ -610,18 +627,27 @@ def __init__( self.rho = self.betas[-1] self.output.iat[-1, self.output.columns.get_loc('var_type')] = 'rho' self.predy_e, self.e_pred, warn = sp_att( - w, self.y, self.predy, yend2[:, -1].reshape(self.n, 1), self.rho - ) + w, self.y, self.predy, yend2[:, -1].reshape(self.n, 1), self.rho, hard_bound=hard_bound) set_warn(self, warn) self.regime_lag_sep = regime_lag_sep self.title = "SPATIAL " + self.title if slx_lags > 0: + for m in self.regimes_set: + r_output = self.output[(self.output['regime'] == str(m)) & (self.output['var_type'] == 'x')] + wx_index = r_output.index[-((len(r_output)-1)//(slx_lags+1)) * slx_lags:] + self.output.loc[wx_index, 'var_type'] = 'wx' self.title = " SPATIAL 2SLS WITH SLX (SPATIAL DURBIN MODEL) - REGIMES" self.other_top = _spat_pseudo_r2(self) + self.slx_lags = slx_lags + diag_out = None if spat_diag: diag_out = _spat_diag_out(self, w, 'yend') - else: - diag_out = None + if spat_impacts and slx_lags == 0: + impacts = _summary_impacts(self, _spmultiplier(w, self.rho)) + try: + diag_out += impacts + except TypeError: + diag_out = impacts output(reg=self, vm=vm, robust=robust, other_end=diag_out, latex=latex) def GM_Lag_Regimes_Multi( @@ -641,6 +667,7 @@ def GM_Lag_Regimes_Multi( gwk=None, sig2n_k=False, cols2regi="all", + spat_impacts=False, spat_diag=False, vm=False, name_y=None, @@ -652,6 +679,7 @@ def GM_Lag_Regimes_Multi( name_gwk=None, name_ds=None, latex=False, + hard_bound=False, ): # pool = mp.Pool(cores) self.name_ds = USER.set_name_ds(name_ds) @@ -771,7 +799,7 @@ def GM_Lag_Regimes_Multi( results[r].y, results[r].predy, results[r].yend[:, -1].reshape(results[r].n, 1), - results[r].rho, + results[r].rho, hard_bound=hard_bound ) set_warn(results[r], warn) results[r].w = w_i[r] @@ -807,12 +835,19 @@ def GM_Lag_Regimes_Multi( ] = results[r].h results[r].other_top = _spat_pseudo_r2(results[r]) results[r].other_mid = "" + if slx_lags > 0: + kx = (results[r].k - results[r].kstar - 1) // (slx_lags + 1) + var_types = ['x'] * (kx + 1) + ['wx'] * kx * slx_lags + ['yend'] * (len(results[r].name_yend) - 1) + ['rho'] + else: + var_types = ['x'] * len(results[r].name_x) + ['yend'] * (len(results[r].name_yend)-1) + ['rho'] + results[r].output = pd.DataFrame({'var_names': results[r].name_x + results[r].name_yend, + 'var_type': var_types, + 'regime': r, 'equation': r}) + self.output = pd.concat([self.output, results[r].output], ignore_index=True) if spat_diag: results[r].other_mid += _spat_diag_out(results[r], results[r].w, 'yend') - self.output = pd.concat([self.output, pd.DataFrame({'var_names': results[r].name_x + results[r].name_yend, - 'var_type': ['x'] * len(results[r].name_x) + [ - 'yend'] * (len(results[r].name_yend)-1) + ['rho'], - 'regime': r, 'equation': r})], ignore_index=True) + if spat_impacts and slx_lags == 0: + results[r].other_mid += _summary_impacts(results[r], _spmultiplier(results[r].w, results[r].rho)) counter += 1 self.multi = results if robust == "hac": @@ -823,9 +858,8 @@ def GM_Lag_Regimes_Multi( "Residuals treated as homoskedastic for the purpose of diagnostics.", ) self.chow = REGI.Chow(self) - if spat_diag: - # self._get_spat_diag_props(y, x, w, yend, q, w_lags, lag_q) - pass + #if spat_diag: + # self._get_spat_diag_props(y, x, w, yend, q, w_lags, lag_q) output(reg=self, vm=vm, robust=robust, other_end=False, latex=latex) def sp_att_reg(self, w_i, regi_ids, wy): @@ -929,9 +963,48 @@ def _work( model.name_h = model.name_x + model.name_q model.name_w = name_w model.name_regimes = name_regimes + model.slx_lags = slx_lags return model +class GM_Lag_Endog_Regimes(GM_Lag_Regimes): + def __init__( + self, y, x, w, n_clusters=None, quorum=-np.inf, trace=True, **kwargs): + + n = USER.check_arrays(y, x) + y = USER.check_y(y, n) + USER.check_weights(w, y, w_required=True) + + # Standardize the variables + x_std = (x - np.mean(x, axis=0)) / np.std(x, axis=0) + + if not n_clusters: + if quorum < 0: + quorum = np.max([(x.shape[1]+1)*10, 30]) + n_clusters_opt = x.shape[0]*0.70//quorum + if n_clusters_opt < 2: + raise ValueError( + "The combination of the values of `N` and `quorum` is not compatible with regimes estimation.") + sk_reg_results = Skater_reg().fit(n_clusters_opt, w, x_std, {'reg':BaseGM_Lag,'y':y,'x':x,'w':w}, quorum=quorum, trace=True) + n_clusters = optim_k([sk_reg_results._trace[i][1][2] for i in range(1, len(sk_reg_results._trace))]) + self.clusters = sk_reg_results._trace[n_clusters-1][0] + else: + try: + # Call the Skater_reg method based on GM_Lag + sk_reg_results = Skater_reg().fit(n_clusters, w, x_std, {'reg':BaseGM_Lag,'y':y,'x':x,'w':w}, quorum=quorum, trace=trace) + self.clusters = sk_reg_results.current_labels_ + except Exception as e: + if str(e) == "one or more input arrays have more columns than rows": + raise ValueError("One or more input ended up with more variables than observations. Please check your setting for `quorum`.") + else: + print("An error occurred:", e) + + self._trace = sk_reg_results._trace + self.SSR = [self._trace[i][1][2] for i in range(1, len(self._trace))] + + GM_Lag_Regimes.__init__(self, y, x, regimes=self.clusters, w=w, name_regimes='Skater_reg', **kwargs) + + def _test(): import doctest diff --git a/spreg/user_output.py b/spreg/user_output.py index cf7f9e68..376fd3dd 100755 --- a/spreg/user_output.py +++ b/spreg/user_output.py @@ -189,12 +189,29 @@ def set_name_q_sp(name_x, w_lags, name_q, lag_q, force_all=False): if lag_q: names = names + name_q sp_inst_names = [] - for j in names: - sp_inst_names.append("W_" + j) - if w_lags > 1: - for i in range(2, w_lags + 1): - for j in names: - sp_inst_names.append("W" + str(i) + "_" + j) + existing_names = set(names) + name_count = {} # Dictionary to store the count of each name + + for name in names: + if not name.startswith("W_"): + name_count[name] = 2 + else: + if name[2:] not in name_count: + name_count[name[2:]] = 2 + + for i in range(w_lags): + for name in names: + if name.startswith("W_"): + lag_name = f"W{name_count[name[2:]]}_{name[2:]}" + name_count[name[2:]] += 1 + else: + if ("W_" + name) not in existing_names: + lag_name = f"W_{name}" + else: + lag_name = f"W{name_count[name]}_{name}" + name_count[name] += 1 + sp_inst_names.append(lag_name) + existing_names.add(lag_name) return sp_inst_names diff --git a/spreg/utils.py b/spreg/utils.py index c369c74d..19911fb0 100755 --- a/spreg/utils.py +++ b/spreg/utils.py @@ -367,13 +367,13 @@ def optim_moments(moments_in, vcX=np.array([0]), all_par=False, start=None): np.array([[float(par[0]), float(par[0]) ** 2.0]]).T, moments ) start = [0.0] - bounds = [(-1.0, 1.0)] + bounds = [(-0.99, 0.99)] if moments[0].shape[0] == 3: optim_par = lambda par: foptim_par( np.array([[float(par[0]), float(par[0]) ** 2.0, float(par[1])]]).T, moments ) start = [0.0, 1.0] - bounds = [(-1.0, 1.0), (0.0, None)] + bounds = [(-0.99, 0.99), (0.0, None)] if moments[0].shape[1] == 4: optim_par = lambda par: foptim_par( np.array( @@ -383,8 +383,9 @@ def optim_moments(moments_in, vcX=np.array([0]), all_par=False, start=None): ) if not start: start = [0.0, 1.0, 1.0] - bounds = [(-1.0, 1.0), (0.0, None), (0.0, None)] + bounds = [(-0.99, 0.99), (0.0, None), (0.0, None)] lambdaX = op.fmin_l_bfgs_b(optim_par, start, approx_grad=True, bounds=bounds) + if all_par: return lambdaX[0] return lambdaX[0][0] @@ -734,7 +735,7 @@ def iter_msg(iteration, max_iter): return iter_stop -def sp_att(w, y, predy, w_y, rho): +def sp_att(w, y, predy, w_y, rho, hard_bound=False): xb = predy - rho * w_y if np.abs(rho) < 1: predy_sp = inverse_prod(w, xb, rho) @@ -742,11 +743,16 @@ def sp_att(w, y, predy, w_y, rho): # Note 1: Here if omitting pseudo-R2; If not, see Note 2. resid_sp = y - predy_sp else: - # warn = "Warning: Estimate for rho is outside the boundary (-1, 1). Computation of true inverse of W was required (slow)." - # predy_sp = inverse_prod(w, xb, rho, inv_method="true_inv") - warn = "*** WARNING: Estimate for spatial lag coefficient is outside the boundary (-1, 1). ***" - predy_sp = np.zeros(y.shape, float) - resid_sp = np.zeros(y.shape, float) + if hard_bound: + raise Exception( + "Spatial autoregressive parameter is outside the maximum/minimum bounds." + ) + else: + # warn = "Warning: Estimate for rho is outside the boundary (-1, 1). Computation of true inverse of W was required (slow)." + # predy_sp = inverse_prod(w, xb, rho, inv_method="true_inv") + warn = "*** WARNING: Estimate for spatial lag coefficient is outside the boundary (-1, 1). ***" + predy_sp = np.zeros(y.shape, float) + resid_sp = np.zeros(y.shape, float) # resid_sp = y - predy_sp #Note 2: Here if computing true inverse; If not, # see Note 1. return predy_sp, resid_sp, warn @@ -789,6 +795,56 @@ def RegressionProps_basic( if vm is not None: reg.vm = vm +def optim_k(trace, window_size=None): + """ + Finds optimal number of regimes for the endogenous spatial regimes model + using a method adapted from Mojena (1977)'s Rule Two. + + Parameters + ---------- + trace : list + List of SSR values for different number of regimes + window_size : integer + Size of the window to be used in the moving average + (Defaults to N//4) + Returns + ------- + i+window_size : integer + Optimal number of regimes + + Examples + -------- + >>> import libpysal as ps + >>> import numpy as np + >>> import spreg + >>> data = ps.io.open(ps.examples.get_path('NAT.dbf')) + >>> y = np.array(data.by_col('HR90')).reshape((-1,1)) + >>> x_var = ['PS90','UE90'] + >>> x = np.array([data.by_col(name) for name in x_var]).T + >>> w = ps.weights.Queen.from_shapefile(ps.examples.get_path("NAT.shp")) + >>> x_std = (x - np.mean(x,axis=0)) / np.std(x,axis=0) + >>> reg = spreg.Skater_reg().fit(20, w, x_std, {'reg':spreg.OLS,'y':y,'x':x}, quorum=100, trace=True) + >>> spreg.utils.optim_k([reg._trace[i][1][2] for i in range(1, len(reg._trace))]) + 9 + + + """ + + N = len(trace) + if not window_size: + window_size = N//4 # Mojena suggests from 70% to 90% + std_dev = [np.std(trace[i:i+window_size]) for i in range(N - window_size + 1)] + ma = np.convolve(trace, np.ones(window_size)/window_size, mode='valid') + treshold = [True] + i = 0 + while treshold[-1] and i < (N - window_size): + b = (6/(window_size*(window_size*window_size-1)) + )*((2*np.sum(np.arange(1, i+2)*trace[window_size-1:i+window_size]) + )-((window_size+1)*np.sum(trace[window_size-1:i+window_size]))) + l = (window_size-1)*b/2 + treshold.append(trace[i+window_size] < ma[i] - b - l - 2.75*std_dev[i]) + i += 1 + return i+window_size def _test(): import doctest From e45c6514ffe20aa3b1748aac86ad6bfc2090cf69 Mon Sep 17 00:00:00 2001 From: Pedro Amaral Date: Mon, 3 Jun 2024 14:38:12 -0300 Subject: [PATCH 02/10] Fixing imports in dgp.py --- spreg/dgp.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/spreg/dgp.py b/spreg/dgp.py index 8f4e34c4..7053e01f 100644 --- a/spreg/dgp.py +++ b/spreg/dgp.py @@ -11,8 +11,9 @@ import math import libpysal from scipy.linalg import expm -import spreg - +from .utils import inverse_prod + + __all__ = [ "make_error", "make_x", @@ -327,7 +328,7 @@ def dgp_errproc(u,w,lam=0.5,model='sar',imethod='power_exp'): print("Error: incompatible weights dimensions") return None if model == 'sar': - y = spreg.utils.inverse_prod(w,u,lam,inv_method=imethod) + y = inverse_prod(w,u,lam,inv_method=imethod) elif model == 'ma': y = u + lam * libpysal.weights.lag_spatial(w,u) else: @@ -468,7 +469,7 @@ def dgp_sperror(u,xb,w,lam=0.5,model='sar',imethod='power_exp'): print("Error: incompatible weights dimensions") return None if model == 'sar': - u1 = spreg.utils.inverse_prod(w,u,lam,inv_method=imethod) + u1 = inverse_prod(w,u,lam,inv_method=imethod) elif model == 'ma': u1 = u + lam * libpysal.weights.lag_spatial(w,u) else: @@ -532,7 +533,7 @@ def dgp_slxerror(u,xb,wxg,w,lam=0.5,model='sar',imethod='power_exp'): print("Error: incompatible weights dimensions") return None if model == 'sar': - u1 = spreg.utils.inverse_prod(w,u,lam,inv_method=imethod) + u1 = inverse_prod(w,u,lam,inv_method=imethod) elif model == 'ma': u1 = u + lam * libpysal.weights.lag_spatial(w,u) else: @@ -587,7 +588,7 @@ def dgp_lag(u,xb,w,rho=0.5,imethod='power_exp'): print("Error: incompatible weights dimensions") return None y1 = xb + u - y = spreg.utils.inverse_prod(w,y1,rho,inv_method=imethod) + y = inverse_prod(w,y1,rho,inv_method=imethod) return y def dgp_spdurbin(u,xb,wxg,w,rho=0.5,imethod='power_exp'): @@ -642,7 +643,7 @@ def dgp_spdurbin(u,xb,wxg,w,rho=0.5,imethod='power_exp'): if w.n != n1: print("Error: incompatible weights dimensions") y1 = xb + wxg + u - y = spreg.utils.inverse_prod(w,y1,rho,inv_method=imethod) + y = inverse_prod(w,y1,rho,inv_method=imethod) return y def dgp_lagerr(u,xb,w,rho=0.5,lam=0.2,model='sar',imethod='power_exp'): @@ -695,14 +696,14 @@ def dgp_lagerr(u,xb,w,rho=0.5,lam=0.2,model='sar',imethod='power_exp'): print("Error: incompatible weights dimensions") return None if model == 'sar': - u1 = spreg.utils.inverse_prod(w,u,lam,inv_method=imethod) + u1 = inverse_prod(w,u,lam,inv_method=imethod) elif model == 'ma': u1 = u + lam * libpysal.weights.lag_spatial(w,u) else: print("Error: unsupported model type") return None y1 = xb + u1 - y = spreg.utils.inverse_prod(w,y1,rho,inv_method=imethod) + y = inverse_prod(w,y1,rho,inv_method=imethod) return y def dgp_gns(u,xb,wxg,w,rho=0.5,lam=0.2,model='sar',imethod='power_exp'): @@ -761,14 +762,14 @@ def dgp_gns(u,xb,wxg,w,rho=0.5,lam=0.2,model='sar',imethod='power_exp'): if w.n != n1: print("Error: incompatible weights dimensions") if model == 'sar': - u1 = spreg.utils.inverse_prod(w,u,lam,inv_method=imethod) + u1 = inverse_prod(w,u,lam,inv_method=imethod) elif model == 'ma': u1 = u + lam * libpysal.weights.lag_spatial(w,u) else: print("Error: unsupported model type") return None y1 = xb + wxg + u1 - y = spreg.utils.inverse_prod(w,y1,rho,inv_method=imethod) + y = inverse_prod(w,y1,rho,inv_method=imethod) return y def dgp_mess(u,xb,w,rho=0.5): From 1d584c15f2d07455163e1818d1ed56ac5348a7cf Mon Sep 17 00:00:00 2001 From: Pedro Amaral Date: Wed, 5 Jun 2024 07:29:34 -0300 Subject: [PATCH 03/10] Updating titles in results for SLX-Error models Updating titles to match upcoming paper --- spreg/error_sp.py | 10 +++++----- spreg/error_sp_het.py | 8 ++++---- spreg/error_sp_het_regimes.py | 12 ++++++------ spreg/error_sp_hom.py | 8 ++++---- spreg/error_sp_hom_regimes.py | 12 ++++++------ spreg/error_sp_regimes.py | 12 ++++++------ spreg/ml_error.py | 4 ++-- spreg/ml_error_regimes.py | 6 +++--- 8 files changed, 36 insertions(+), 36 deletions(-) diff --git a/spreg/error_sp.py b/spreg/error_sp.py index 406ac155..cdbf6275 100644 --- a/spreg/error_sp.py +++ b/spreg/error_sp.py @@ -135,7 +135,7 @@ class GM_Error(BaseGM_Error): Spatial weights object (always needed) slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. vm : boolean If True, include variance-covariance matrix in summary results @@ -307,7 +307,7 @@ def __init__( x_constant = np.hstack((x_constant, lag_x)) # name_x += USER.set_name_spatial_lags(name_x, slx_lags) name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant - self.title += " WITH SLX (SDEM)" + self.title += " WITH SLX (SLX-Error)" BaseGM_Error.__init__(self, y=y, x=x_constant, w=w.sparse, hard_bound=hard_bound) self.name_ds = USER.set_name_ds(name_ds) @@ -459,7 +459,7 @@ class GM_Endog_Error(BaseGM_Endog_Error): Spatial weights object (always needed) slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. vm : boolean If True, include variance-covariance matrix in summary results @@ -671,7 +671,7 @@ def __init__( x_constant = np.hstack((x_constant, lag_x)) # name_x += USER.set_name_spatial_lags(name_x, slx_lags) name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant - self.title += " WITH SLX (SDEM)" + self.title += " WITH SLX (SLX-Error)" BaseGM_Endog_Error.__init__(self, y=y, x=x_constant, w=w.sparse, yend=yend, q=q, hard_bound=hard_bound) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) @@ -1029,7 +1029,7 @@ class GMM_Error(GM_Error, GM_Endog_Error, GM_Combo, GM_Error_Het, GM_Endog_Error If True, then a spatial lag of the dependent variable is included. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM or GNSM type. + If slx_lags>0, the specification becomes of the SLX-Error or GNSM type. vm : boolean If True, include variance-covariance matrix in summary results diff --git a/spreg/error_sp_het.py b/spreg/error_sp_het.py index ac2bc598..abac7638 100755 --- a/spreg/error_sp_het.py +++ b/spreg/error_sp_het.py @@ -188,7 +188,7 @@ class GM_Error_Het(BaseGM_Error_Het): Spatial weights object slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional @@ -386,7 +386,7 @@ def __init__( x_constant = np.hstack((x_constant, lag_x)) # name_x += USER.set_name_spatial_lags(name_x, slx_lags) name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # no constant - self.title += " WITH SLX (SDEM)" + self.title += " WITH SLX (SLX-Error)" BaseGM_Error_Het.__init__( self, y, @@ -673,7 +673,7 @@ class GM_Endog_Error_Het(BaseGM_Endog_Error_Het): Spatial weights object slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional @@ -913,7 +913,7 @@ def __init__( lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # no constant - self.title += " WITH SLX (SDEM)" + self.title += " WITH SLX (SLX-Error)" BaseGM_Endog_Error_Het.__init__( self, y=y, diff --git a/spreg/error_sp_het_regimes.py b/spreg/error_sp_het_regimes.py index c5c79100..9ba5b68e 100644 --- a/spreg/error_sp_het_regimes.py +++ b/spreg/error_sp_het_regimes.py @@ -67,7 +67,7 @@ class GM_Error_Het_Regimes(RegressionPropsY, REGI.Regimes_Frame): Always False, kept for consistency, ignored. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from Arraiz et al. Note: epsilon provides an additional stop condition. @@ -438,7 +438,7 @@ def __init__( if slx_lags == 0: self.title = "GM SPATIALLY WEIGHTED MODEL (HET) - REGIMES" else: - self.title = "GM SPATIALLY WEIGHTED MODEL + SLX (SDEM-HET) - REGIMES" + self.title = "GM SPATIALLY WEIGHTED MODEL + SLX (Error-HET) - REGIMES" self.name_x.append("lambda") self.kf += 1 @@ -626,7 +626,7 @@ class GM_Endog_Error_Het_Regimes(RegressionPropsY, REGI.Regimes_Frame): Always False, kept for consistency, ignored. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional stop condition. @@ -1121,7 +1121,7 @@ def __init__( if slx_lags == 0: self.title = ("GM SPATIALLY WEIGHTED 2SLS (HET) - REGIMES") else: - self.title = ("GM SPATIALLY WEIGHTED 2SLS + SLX (SDEM-HET) - REGIMES") + self.title = ("GM SPATIALLY WEIGHTED 2SLS + SLX (Error-HET) - REGIMES") output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) def _endog_error_het_regimes_multi( @@ -1837,7 +1837,7 @@ def _work_error( if slx_lags == 0: model.title = "GM SPATIALLY WEIGHTED LS (HET) - REGIME %s" % r else: - model.title = "GM SPATIALLY WEIGHTED LS + SLX (SDEM-HET) - REGIME %s" % r + model.title = "GM SPATIALLY WEIGHTED LS + SLX (Error-HET) - REGIME %s" % r model.name_ds = name_ds model.name_y = "%s_%s" % (str(r), name_y) model.name_x = ["%s_%s" % (str(r), i) for i in name_x] @@ -1910,7 +1910,7 @@ def _work_endog_error( if add_lag != False: model.title = "GM SPATIAL COMBO MODEL + SLX (GNSM-HET) - REGIME %s" % r else: - model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (SDEM-HET) - REGIME %s" % r + model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (Error-HET) - REGIME %s" % r model.name_ds = name_ds model.name_y = "%s_%s" % (str(r), name_y) model.name_x = ["%s_%s" % (str(r), i) for i in name_x] diff --git a/spreg/error_sp_hom.py b/spreg/error_sp_hom.py index e51f4ea2..cfa78852 100644 --- a/spreg/error_sp_hom.py +++ b/spreg/error_sp_hom.py @@ -182,7 +182,7 @@ class GM_Error_Hom(BaseGM_Error_Hom): Spatial weights object slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional stop condition. @@ -376,7 +376,7 @@ def __init__( x_constant = np.hstack((x_constant, lag_x)) # name_x += USER.set_name_spatial_lags(name_x, slx_lags) name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant - self.title += " WITH SLX (SDEM)" + self.title += " WITH SLX (SLX-Error)" BaseGM_Error_Hom.__init__( self, y=y, @@ -595,7 +595,7 @@ class GM_Endog_Error_Hom(BaseGM_Endog_Error_Hom): Spatial weights object slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional stop condition. @@ -834,7 +834,7 @@ def __init__( x_constant = np.hstack((x_constant, lag_x)) # name_x += USER.set_name_spatial_lags(name_x, slx_lags) name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant - self.title += " WITH SLX (SDEM)" + self.title += " WITH SLX (SLX-Error)" BaseGM_Endog_Error_Hom.__init__( self, y=y, diff --git a/spreg/error_sp_hom_regimes.py b/spreg/error_sp_hom_regimes.py index b4eba259..0853e2db 100644 --- a/spreg/error_sp_hom_regimes.py +++ b/spreg/error_sp_hom_regimes.py @@ -69,7 +69,7 @@ class GM_Error_Hom_Regimes(RegressionPropsY, REGI.Regimes_Frame): Always False, kept for consistency, ignored. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional @@ -450,7 +450,7 @@ def __init__( if slx_lags == 0: self.title = "GM SPATIALLY WEIGHTED MODEL (HOM) - REGIMES" else: - self.title = "GM SPATIALLY WEIGHTED MODEL + SLX (SDEM-HOM) - REGIMES" + self.title = "GM SPATIALLY WEIGHTED MODEL + SLX (Error-HOM) - REGIMES" self.name_x.append("lambda") self.kf += 1 self.chow = REGI.Chow(self) @@ -638,7 +638,7 @@ class GM_Endog_Error_Hom_Regimes(RegressionPropsY, REGI.Regimes_Frame): Always False, kept for consistency, ignored. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional stop condition. @@ -1116,7 +1116,7 @@ def __init__( if slx_lags == 0: self.title = ("GM SPATIALLY WEIGHTED 2SLS (HOM) - REGIMES") else: - self.title = ("GM SPATIALLY WEIGHTED 2SLS WITH SLX (SDEM-HOM) - REGIMES") + self.title = ("GM SPATIALLY WEIGHTED 2SLS WITH SLX (Error-HOM) - REGIMES") output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) def _endog_error_hom_regimes_multi( @@ -1835,7 +1835,7 @@ def _work_error( if slx_lags == 0: model.title = "GM SPATIALLY WEIGHTED 2SLS (HOM) - REGIME %s" % r else: - model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (SDEM-HOM) - REGIME %s" % r + model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (Error-HOM) - REGIME %s" % r model.name_ds = name_ds model.name_y = "%s_%s" % (str(r), name_y) model.name_x = ["%s_%s" % (str(r), i) for i in name_x] @@ -1898,7 +1898,7 @@ def _work_endog_error( if add_lag != False: model.title = "GM SPATIAL COMBO MODEL + SLX (GNSM-HOM) - REGIME %s" % r else: - model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (SDEM-HOM) - REGIME %s" % r + model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (Error-HOM) - REGIME %s" % r model.name_ds = name_ds model.name_y = "%s_%s" % (str(r), name_y) model.name_x = ["%s_%s" % (str(r), i) for i in name_x] diff --git a/spreg/error_sp_regimes.py b/spreg/error_sp_regimes.py index 7dbe978e..5c9ddd5d 100644 --- a/spreg/error_sp_regimes.py +++ b/spreg/error_sp_regimes.py @@ -60,7 +60,7 @@ class GM_Error_Regimes(RegressionPropsY, REGI.Regimes_Frame): Always False, kept for consistency, ignored. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. vm : boolean If True, include variance-covariance matrix in summary results @@ -366,7 +366,7 @@ def __init__( if slx_lags == 0: self.title = "GM SPATIALLY WEIGHTED MODEL - REGIMES" else: - self.title = "GM SPATIALLY WEIGHTED MODEL + SLX (SDEM) - REGIMES" + self.title = "GM SPATIALLY WEIGHTED MODEL + SLX (SLX-Error) - REGIMES" self.name_x.append("lambda") self.kf += 1 self.chow = REGI.Chow(self) @@ -540,7 +540,7 @@ class GM_Endog_Error_Regimes(RegressionPropsY, REGI.Regimes_Frame): Always False, kept for consistency, ignored. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. vm : boolean If True, include variance-covariance matrix in summary results @@ -949,7 +949,7 @@ def __init__( if slx_lags == 0: self.title = ("GM SPATIALLY WEIGHTED 2SLS - REGIMES") else: - self.title = ("GM SPATIALLY WEIGHTED 2SLS WITH SLX (SDEM) - REGIMES") + self.title = ("GM SPATIALLY WEIGHTED 2SLS WITH SLX (SLX-Error) - REGIMES") output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) @@ -1640,7 +1640,7 @@ class GMM_Error_Regimes(GM_Error_Regimes, GM_Combo_Regimes, GM_Endog_Error_Regim If True, then a spatial lag of the dependent variable is included. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM or GNSM type. + If slx_lags>0, the specification becomes of the SLX-Error or GNSM type. vm : boolean If True, include variance-covariance matrix in summary results @@ -2029,7 +2029,7 @@ def _work_endog_error( if add_lag != False: model.title = "GM SPATIAL COMBO MODEL + SLX (GNSM) - REGIME %s" % r else: - model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (SDEM) - REGIME %s" % r + model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (SLX-Error) - REGIME %s" % r model.name_ds = name_ds model.name_y = "%s_%s" % (str(r), name_y) model.name_x = ["%s_%s" % (str(r), i) for i in name_x] diff --git a/spreg/ml_error.py b/spreg/ml_error.py index 0f6017d5..aad98fab 100644 --- a/spreg/ml_error.py +++ b/spreg/ml_error.py @@ -313,7 +313,7 @@ class ML_Error(BaseML_Error): Spatial weights sparse matrix slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. method : string if 'full', brute force calculation (full matrix expressions) if 'ord', Ord eigenvalue method @@ -493,7 +493,7 @@ def __init__( x_constant = np.hstack((x_constant, lag_x)) # name_x += USER.set_name_spatial_lags(name_x, slx_lags) name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant from name_x - self.title += " WITH SLX (SDEM)" + self.title += " WITH SLX (SLX-Error)" self.title += " (METHOD = " + method + ")" method = method.upper() diff --git a/spreg/ml_error_regimes.py b/spreg/ml_error_regimes.py index 0b315af4..9793b9df 100644 --- a/spreg/ml_error_regimes.py +++ b/spreg/ml_error_regimes.py @@ -54,7 +54,7 @@ class ML_Error_Regimes(BaseML_Error, REGI.Regimes_Frame): Spatial weights sparse matrix slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. method : string if 'full', brute force calculation (full matrix expressions) if 'ord', Ord eigenvalue computation @@ -378,7 +378,7 @@ def __init__( self.title = "ML SPATIAL ERROR" if slx_lags >0: - self.title += " WITH SLX (SDEM)" + self.title += " WITH SLX (SLX-Error)" self.title += " - REGIMES (METHOD = " + method + ")" self.name_x = USER.set_name_x(name_x, x, constant=True) @@ -527,7 +527,7 @@ def _work_error( model.w = w_r model.title = "ML SPATIAL ERROR" if slx_lags >0: - model.title += " WITH SLX (SDEM)" + model.title += " WITH SLX (SLX-Error)" model.title += " - REGIME " + str(r) + " (METHOD = " + method + ")" model.name_ds = name_ds model.name_y = "%s_%s" % (str(r), name_y) From 22cf25fcb4212294e7ab8f4b742b629f4e89da06 Mon Sep 17 00:00:00 2001 From: Pedro Amaral Date: Tue, 18 Jun 2024 19:25:32 -0300 Subject: [PATCH 04/10] Setting spat_diag default to True in ml_lag --- spreg/ml_lag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spreg/ml_lag.py b/spreg/ml_lag.py index 2ebc3d45..5d350635 100755 --- a/spreg/ml_lag.py +++ b/spreg/ml_lag.py @@ -596,7 +596,7 @@ def __init__( epsilon=0.0000001, spat_impacts=True, vm=False, - spat_diag=False, + spat_diag=True, name_y=None, name_x=None, name_w=None, From 4fa1f671d6be1460bde32dc8490f29c1ef1dff4f Mon Sep 17 00:00:00 2001 From: Pedro Amaral Date: Mon, 24 Jun 2024 16:53:05 -0300 Subject: [PATCH 05/10] Changing test_ml_error db to try to fix CI failures --- spreg/tests/test_ml_error.py | 130 ++++++++++++++++++----------------- 1 file changed, 67 insertions(+), 63 deletions(-) diff --git a/spreg/tests/test_ml_error.py b/spreg/tests/test_ml_error.py index de9b051b..cf3a8605 100644 --- a/spreg/tests/test_ml_error.py +++ b/spreg/tests/test_ml_error.py @@ -1,7 +1,5 @@ import unittest import libpysal -from libpysal.examples import load_example -from libpysal.weights import Queen import numpy as np from scipy import sparse from spreg.ml_error import ML_Error @@ -14,14 +12,17 @@ class TestMLError(unittest.TestCase): def setUp(self): - south = load_example("South") - db = libpysal.io.open(south.get_path("south.dbf"), "r") - self.y_name = "HR90" - self.y = np.array(db.by_col(self.y_name)) + db = libpysal.io.open(libpysal.examples.get_path("baltim.dbf"), "r") + self.ds_name = "baltim.dbf" + self.y_name = "PRICE" + self.y = np.array(db.by_col(self.y_name)).T self.y.shape = (len(self.y), 1) - self.x_names = ["RD90", "PS90", "UE90", "DV90"] + self.x_names = ["NROOM", "AGE", "SQFT"] self.x = np.array([db.by_col(var) for var in self.x_names]).T - self.w = Queen.from_shapefile(south.get_path("south.shp")) + ww = libpysal.io.open(libpysal.examples.get_path("baltim_q.gal")) + self.w = ww.read() + ww.close() + self.w_name = "baltim_q.gal" self.w.transform = "r" def _estimate_and_compare(self, method="FULL", RTOL=RTOL): @@ -31,57 +32,58 @@ def _estimate_and_compare(self, method="FULL", RTOL=RTOL): w=self.w, name_y=self.y_name, name_x=self.x_names, - name_w="south_q.gal", + name_w=self.w_name, method=method, ) - betas = np.array([[6.1492], [4.4024], [1.7784], [-0.3781], [0.4858], [0.2991]]) + betas = np.array([[19.45930348], + [ 3.98928064], + [-0.16714232], + [ 0.57336871], + [ 0.71757002]]) np.testing.assert_allclose(reg.betas, betas, RTOL + 0.0001) - u = np.array([-5.97649777]) + u = np.array([29.870239]) np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([6.92258051]) + predy = np.array([17.129761]) np.testing.assert_allclose(reg.predy[0], predy, RTOL) - n = 1412 + n = 211 np.testing.assert_allclose(reg.n, n, RTOL) - k = 5 + k = 4 np.testing.assert_allclose(reg.k, k, RTOL) - y = np.array([0.94608274]) + y = np.array([47.]) np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, -0.39902838, 0.89645344, 6.85780705, 7.2636377]) + x = np.array([ 1. , 4. , 148. , 11.25]) np.testing.assert_allclose(reg.x[0], x, RTOL) - e = np.array([-4.92843327]) + e = np.array([44.392043]) np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) - my = 9.5492931620846928 + my = 44.30718 np.testing.assert_allclose(reg.mean_y, my) - sy = 7.0388508798387219 + sy = 23.606077 np.testing.assert_allclose(reg.std_y, sy) vm = np.array( - [1.06476526, 0.05548248, 0.04544514, 0.00614425, 0.01481356, 0.00143001] + [3.775969e+01, 1.337534e+00, 4.440495e-03, 2.890193e-02, + 3.496050e-03] ) np.testing.assert_allclose(reg.vm.diagonal(), vm, RTOL) - sig2 = np.array([[32.40685441]]) + sig2 = np.array([[219.239799]]) np.testing.assert_allclose(reg.sig2, sig2, RTOL) - pr2 = 0.3057664820364818 - np.testing.assert_allclose(reg.pr2, pr2) + pr2 = 0.341471 + np.testing.assert_allclose(reg.pr2, pr2, RTOL) std_err = np.array( - [1.03187463, 0.23554719, 0.21317867, 0.07838525, 0.12171098, 0.03781546] + [6.144892, 1.156518, 0.066637, 0.170006, 0.059127] ) np.testing.assert_allclose(reg.std_err, std_err, RTOL) - z_stat = [ - (5.9592751097983534, 2.5335926307459251e-09), - (18.690182928021841, 5.9508619446611137e-78), - (8.3421632936950338, 7.2943630281051907e-17), - (-4.8232686291115678, 1.4122456582517099e-06), - (3.9913060809142995, 6.5710406838016854e-05), - (7.9088780724028922, 2.5971882547279339e-15), - ] + z_stat = [(3.166744811610107, 0.0015415552994677963), + (3.4493895324306485, 0.0005618555635414317), + (-2.5082495410045618, 0.012133094835810014), + (3.3726442232925864, 0.0007445008419860677), + (12.13599679437352, 6.807593113579489e-34)] np.testing.assert_allclose(reg.z_stat, z_stat, RTOL, atol=ATOL) - logll = -4471.407066887894 + logll = -881.269405 np.testing.assert_allclose(reg.logll, logll, RTOL) - aic = 8952.8141337757879 + aic = 1770.538809 np.testing.assert_allclose(reg.aic, aic, RTOL) - schwarz = 8979.0779458660545 + schwarz = 1783.946242 np.testing.assert_allclose(reg.schwarz, schwarz, RTOL) - def test_dense(self): self._estimate_and_compare(method="FULL") @@ -95,55 +97,57 @@ def test_ord(self): w=self.w, name_y=self.y_name, name_x=self.x_names, - name_w="south_q.gal", + name_w=self.w_name, method="ORD", ) - betas = np.array([[6.1492], [4.4024], [1.7784], [-0.3781], [0.4858], [0.2991]]) - np.testing.assert_allclose(reg.betas, betas, RTOL + 0.0001) - u = np.array([-5.97649777]) + betas = np.array([[19.45930348], + [ 3.98928064], + [-0.16714232], + [ 0.57336871], + [ 0.71757002]]) + np.testing.assert_allclose(reg.betas, betas, RTOL) + u = np.array([29.870239]) np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([6.92258051]) + predy = np.array([17.129761]) np.testing.assert_allclose(reg.predy[0], predy, RTOL) - n = 1412 + n = 211 np.testing.assert_allclose(reg.n, n, RTOL) - k = 5 + k = 4 np.testing.assert_allclose(reg.k, k, RTOL) - y = np.array([0.94608274]) + y = np.array([47.]) np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, -0.39902838, 0.89645344, 6.85780705, 7.2636377]) + x = np.array([ 1. , 4. , 148. , 11.25]) np.testing.assert_allclose(reg.x[0], x, RTOL) - e = np.array([-4.92843327]) + e = np.array([44.392043]) np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) - my = 9.5492931620846928 + my = 44.30718 np.testing.assert_allclose(reg.mean_y, my) - sy = 7.0388508798387219 + sy = 23.606077 np.testing.assert_allclose(reg.std_y, sy) vm = np.array( - [1.06476526, 0.05548248, 0.04544514, 0.00614425, 0.01481356, 0.001501] + [3.775969e+01, 1.337534e+00, 4.440495e-03, 2.890193e-02, + 3.586781e-03] ) np.testing.assert_allclose(reg.vm.diagonal(), vm, RTOL * 10) - sig2 = np.array([[32.40685441]]) + sig2 = np.array([[219.239799]]) np.testing.assert_allclose(reg.sig2, sig2, RTOL) - pr2 = 0.3057664820364818 + pr2 = 0.34147059826596426 np.testing.assert_allclose(reg.pr2, pr2) std_err = np.array( - [1.03187463, 0.23554719, 0.21317867, 0.07838525, 0.12171098, 0.038744] + [6.144892, 1.156518, 0.066637, 0.170006, 0.05989 ] ) np.testing.assert_allclose(reg.std_err, std_err, RTOL * 10) - z_stat = [ - (5.95927510, 2.5335927e-09), - (18.6901829, 5.9508630e-78), - (8.34216329, 7.2943634e-17), - (-4.8232686, 1.4122457e-06), - (3.99130608, 6.5710407e-05), - (7.71923784, 1.1702739e-14), - ] + z_stat = [(3.166744811610107, 0.0015415552994677963), + (3.4493895324306485, 0.0005618555635414317), + (-2.5082495410045618, 0.012133094835810014), + (3.3726442232925864, 0.0007445008419860677), + (11.981517603949666, 4.441183328428627e-33)] np.testing.assert_allclose(reg.z_stat, z_stat, rtol=RTOL, atol=ATOL) - logll = -4471.407066887894 + logll = -881.269405 np.testing.assert_allclose(reg.logll, logll, RTOL) - aic = 8952.8141337757879 + aic = 1770.538809 np.testing.assert_allclose(reg.aic, aic, RTOL) - schwarz = 8979.0779458660545 + schwarz = 1783.946242 np.testing.assert_allclose(reg.schwarz, schwarz, RTOL) From b4df185609227ba3fd96cc0492152b70ab10df71 Mon Sep 17 00:00:00 2001 From: Pedro Amaral Date: Mon, 24 Jun 2024 18:38:31 -0300 Subject: [PATCH 06/10] Updating utils.py and a few tests tolerances --- spreg/error_sp_het.py | 6 - spreg/ml_lag_regimes.py | 2 + spreg/tests/test_error_sp_het.py | 769 +++++++++-------------- spreg/tests/test_error_sp_het_regimes.py | 667 +++++++------------- spreg/tests/test_error_sp_het_sparse.py | 769 +++++++++-------------- spreg/tests/test_sp_panels.py | 239 ++----- spreg/utils.py | 11 +- 7 files changed, 904 insertions(+), 1559 deletions(-) diff --git a/spreg/error_sp_het.py b/spreg/error_sp_het.py index abac7638..24c9adcb 100755 --- a/spreg/error_sp_het.py +++ b/spreg/error_sp_het.py @@ -129,12 +129,6 @@ def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, step1c=False, hard_boun else: lambda2 = lambda1 - # Forcing the 1st step lambda to be in the range [-0.9, 0.9] to avoid perfect collinearity in step 2 in case of SLX-Error or GNS models - #if lambda2 > 0.9: - # lambda_old = 0.9 - #elif lambda2 < -0.9: - # lambda_old = -0.9 - #else: lambda_old = lambda2 self.iteration, eps = 0, 1 diff --git a/spreg/ml_lag_regimes.py b/spreg/ml_lag_regimes.py index bfed43f1..93da512f 100644 --- a/spreg/ml_lag_regimes.py +++ b/spreg/ml_lag_regimes.py @@ -643,6 +643,8 @@ def _work( model.aic = DIAG.akaike(reg=model) model.schwarz = DIAG.schwarz(reg=model) model.slx_lags = slx_lags + model.w = w_r + model.rho = model.betas[-1] return model diff --git a/spreg/tests/test_error_sp_het.py b/spreg/tests/test_error_sp_het.py index 8b4e6e78..727c2951 100644 --- a/spreg/tests/test_error_sp_het.py +++ b/spreg/tests/test_error_sp_het.py @@ -5,219 +5,191 @@ from libpysal.common import RTOL import spreg - class TestBaseGMErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.X = np.hstack((np.ones(self.y.shape), self.X)) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.X = np.hstack((np.ones(self.y.shape),self.X)) + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): reg = HET.BaseGM_Error_Het(self.y, self.X, self.w.sparse, step1c=True) - betas = np.array([[47.99626638], [0.71048989], [-0.55876126], [0.41178776]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([27.38122697]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([32.29765975]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([53.08577603]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + betas = np.array([[ 47.99626638], [ 0.71048989], [ -0.55876126], [ 0.41178776]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 27.38122697]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 32.29765975]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 53.08577603]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0], x, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0],x,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [1.31767529e02, -3.58368748e00, -1.65090647e00, 0.00000000e00], - [-3.58368748e00, 1.35513711e-01, 3.77539055e-02, 0.00000000e00], - [-1.65090647e00, 3.77539055e-02, 2.61042702e-02, 0.00000000e00], - [0.00000000e00, 0.00000000e00, 0.00000000e00, 2.82398517e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) - xtx = np.array( - [ - [4.90000000e01, 7.04371999e02, 1.72131237e03], - [7.04371999e02, 1.16866734e04, 2.15575320e04], - [1.72131237e03, 2.15575320e04, 7.39058986e04], - ] - ) - np.testing.assert_allclose(reg.xtx, xtx, RTOL) - - + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 1.31767529e+02, -3.58368748e+00, -1.65090647e+00, + 0.00000000e+00], + [ -3.58368748e+00, 1.35513711e-01, 3.77539055e-02, + 0.00000000e+00], + [ -1.65090647e+00, 3.77539055e-02, 2.61042702e-02, + 0.00000000e+00], + [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, + 2.82398517e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) + xtx = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04]]) + np.testing.assert_allclose(reg.xtx,xtx,RTOL) + class TestGMErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): reg = HET.GM_Error_Het(self.y, self.X, self.w, step1c=True) - betas = np.array([[47.99626638], [0.71048989], [-0.55876126], [0.41178776]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([27.38122697]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([32.29765975]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([53.08577603]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + betas = np.array([[ 47.99626638], [ 0.71048989], [ -0.55876126], [ 0.41178776]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 27.38122697]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 32.29765975]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 53.08577603]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0], x, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0],x,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [1.31767529e02, -3.58368748e00, -1.65090647e00, 0.00000000e00], - [-3.58368748e00, 1.35513711e-01, 3.77539055e-02, 0.00000000e00], - [-1.65090647e00, 3.77539055e-02, 2.61042702e-02, 0.00000000e00], - [0.00000000e00, 0.00000000e00, 0.00000000e00, 2.82398517e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 1.31767529e+02, -3.58368748e+00, -1.65090647e+00, + 0.00000000e+00], + [ -3.58368748e+00, 1.35513711e-01, 3.77539055e-02, + 0.00000000e+00], + [ -1.65090647e+00, 3.77539055e-02, 2.61042702e-02, + 0.00000000e+00], + [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, + 2.82398517e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) pr2 = 0.34951013222581306 - np.testing.assert_allclose(reg.pr2, pr2) - stde = np.array([11.47900385, 0.36812187, 0.16156816, 0.16804717]) - np.testing.assert_allclose(reg.std_err, stde, RTOL) - z_stat = np.array( - [ - [4.18122226e00, 2.89946274e-05], - [1.93003988e00, 5.36018970e-02], - [-3.45836247e00, 5.43469673e-04], - [2.45042960e00, 1.42685863e-02], - ] - ) - np.testing.assert_allclose(reg.z_stat, z_stat, RTOL) - xtx = np.array( - [ - [4.90000000e01, 7.04371999e02, 1.72131237e03], - [7.04371999e02, 1.16866734e04, 2.15575320e04], - [1.72131237e03, 2.15575320e04, 7.39058986e04], - ] - ) - np.testing.assert_allclose(reg.xtx, xtx, RTOL) - + np.testing.assert_allclose(reg.pr2,pr2) + stde = np.array([ 11.47900385, 0.36812187, 0.16156816, 0.16804717]) + np.testing.assert_allclose(reg.std_err,stde,RTOL) + z_stat = np.array([[ 4.18122226e+00, 2.89946274e-05], + [ 1.93003988e+00, 5.36018970e-02], + [ -3.45836247e+00, 5.43469673e-04], + [ 2.45042960e+00, 1.42685863e-02]]) + np.testing.assert_allclose(reg.z_stat,z_stat,RTOL) + xtx = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04]]) + np.testing.assert_allclose(reg.xtx,xtx,RTOL) class TestBaseGMEndogErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) yd = [] yd.append(db.by_col("CRIME")) self.yd = np.array(yd).T q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HET.BaseGM_Endog_Error_Het( - self.y, self.X, self.yd, self.q, self.w.sparse, step1c=True - ) - betas = np.array([[55.39707924], [0.46563046], [-0.67038326], [0.41135023]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([26.51812895]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([31.46604707]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([53.94887405]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + reg = HET.BaseGM_Endog_Error_Het(self.y, self.X, self.yd, self.q, self.w.sparse, step1c=True) + betas = np.array([[ 55.39707924], [ 0.46563046], [ -0.67038326], [ 0.41135023]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 26.51812895]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 31.46604707]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 53.94887405]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531]) - np.testing.assert_allclose(reg.x[0], x, RTOL) - yend = np.array([15.72598]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([5.03]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.z[0], z, RTOL) - h = np.array([1.0, 19.531, 5.03]) - np.testing.assert_allclose(reg.h[0], h, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531]) + np.testing.assert_allclose(reg.x[0],x,RTOL) + yend = np.array([ 15.72598]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 5.03]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.z[0],z,RTOL) + h = np.array([ 1. , 19.531, 5.03 ]) + np.testing.assert_allclose(reg.h[0],h,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [8.34637805e02, -2.16932259e01, -1.33327894e01, 1.65840848e00], - [-2.16932259e01, 5.97683070e-01, 3.39503523e-01, -3.90111107e-02], - [-1.33327894e01, 3.39503523e-01, 2.19008080e-01, -2.81929695e-02], - [1.65840848e00, -3.90111107e-02, -2.81929695e-02, 3.15686105e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) - hth = np.array( - [ - [49.0, 704.371999, 139.75], - [704.371999, 11686.67338121, 2246.12800625], - [139.75, 2246.12800625, 498.5851], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - - + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 8.34637805e+02, -2.16932259e+01, -1.33327894e+01, + 1.65840848e+00], + [ -2.16932259e+01, 5.97683070e-01, 3.39503523e-01, + -3.90111107e-02], + [ -1.33327894e+01, 3.39503523e-01, 2.19008080e-01, + -2.81929695e-02], + [ 1.65840848e+00, -3.90111107e-02, -2.81929695e-02, + 3.15686105e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) + hth = np.array([[ 49. , 704.371999 , 139.75 ], + [ 704.371999 , 11686.67338121, 2246.12800625], + [ 139.75 , 2246.12800625, 498.5851 ]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) + class TestGMEndogErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T @@ -227,364 +199,211 @@ def setUp(self): q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HET.GM_Endog_Error_Het( - self.y, self.X, self.yd, self.q, self.w, step1c=True - ) - betas = np.array([[55.39707924], [0.46563046], [-0.67038326], [0.41135023]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([26.51812895]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([53.94887405]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + reg = HET.GM_Endog_Error_Het(self.y, self.X, self.yd, self.q, self.w, step1c=True) + betas = np.array([[ 55.39707924], [ 0.46563046], [ -0.67038326], [ 0.41135023]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 26.51812895]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + predy = np.array([ 53.94887405]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531]) - np.testing.assert_allclose(reg.x[0], x, RTOL) - yend = np.array([15.72598]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([5.03]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.z[0], z, RTOL) - h = np.array([1.0, 19.531, 5.03]) - np.testing.assert_allclose(reg.h[0], h, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531]) + np.testing.assert_allclose(reg.x[0],x,RTOL) + yend = np.array([ 15.72598]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 5.03]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.z[0],z,RTOL) + h = np.array([ 1. , 19.531, 5.03 ]) + np.testing.assert_allclose(reg.h[0],h,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [8.34637805e02, -2.16932259e01, -1.33327894e01, 1.65840848e00], - [-2.16932259e01, 5.97683070e-01, 3.39503523e-01, -3.90111107e-02], - [-1.33327894e01, 3.39503523e-01, 2.19008080e-01, -2.81929695e-02], - [1.65840848e00, -3.90111107e-02, -2.81929695e-02, 3.15686105e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 8.34637805e+02, -2.16932259e+01, -1.33327894e+01, + 1.65840848e+00], + [ -2.16932259e+01, 5.97683070e-01, 3.39503523e-01, + -3.90111107e-02], + [ -1.33327894e+01, 3.39503523e-01, 2.19008080e-01, + -2.81929695e-02], + [ 1.65840848e+00, -3.90111107e-02, -2.81929695e-02, + 3.15686105e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) pr2 = 0.34648011338954804 - np.testing.assert_allclose(reg.pr2, pr2, RTOL) - std_err = np.array([28.89009873, 0.77309965, 0.46798299, 0.17767558]) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - z_stat = np.array( - [ - (1.9175109006819244, 0.055173057472126787), - (0.60229035155742305, 0.54698088217644414), - (-1.4324949211864271, 0.15200223057569454), - (2.3151759776869496, 0.020603303355572443), - ] - ) - np.testing.assert_allclose(reg.z_stat, z_stat, RTOL) - hth = np.array( - [ - [49.0, 704.371999, 139.75], - [704.371999, 11686.67338121, 2246.12800625], - [139.75, 2246.12800625, 498.5851], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - - + np.testing.assert_allclose(reg.pr2,pr2,RTOL) + std_err = np.array([ 28.89009873, 0.77309965, 0.46798299, + 0.17767558]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + z_stat = np.array([(1.9175109006819244, 0.055173057472126787), (0.60229035155742305, 0.54698088217644414), (-1.4324949211864271, 0.15200223057569454), (2.3151759776869496, 0.020603303355572443)]) + np.testing.assert_allclose(reg.z_stat,z_stat,RTOL) + hth = np.array([[ 49. , 704.371999 , 139.75 ], + [ 704.371999 , 11686.67338121, 2246.12800625], + [ 139.75 , 2246.12800625, 498.5851 ]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) + class TestBaseGMComboHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): # Only spatial lag yd2, q2 = spreg.utils.set_endog(self.y, self.X, self.w, None, None, 1, True) - self.X = np.hstack((np.ones(self.y.shape), self.X)) - reg = HET.BaseGM_Combo_Het( - self.y, self.X, yend=yd2, q=q2, w=self.w.sparse, step1c=True - ) - betas = np.array( - [[57.7778574], [0.73034922], [-0.59257362], [-0.2230231], [0.56636724]] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([25.65156033]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([31.87664403]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([54.81544267]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + self.X = np.hstack((np.ones(self.y.shape),self.X)) + reg = HET.BaseGM_Combo_Het(self.y, self.X, yend=yd2, q=q2, w=self.w.sparse, step1c=True) + betas = np.array([[ 57.7778574 ], [ 0.73034922], [ -0.59257362], [ -0.2230231 ], [ 0.56636724]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 25.65156033]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 31.87664403]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 54.81544267]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 4 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0], x, RTOL) - yend = np.array([35.4585005]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([18.594, 24.7142675]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598, 35.4585005]) - np.testing.assert_allclose(reg.z[0], z, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0],x,RTOL) + yend = np.array([ 35.4585005]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 18.594 , 24.7142675]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598 , 35.4585005]) + np.testing.assert_allclose(reg.z[0],z,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy, RTOL) - vm = np.array( - [ - [ - 4.86218274e02, - -2.77268729e00, - -1.59987770e00, - -1.01969471e01, - 2.74302006e00, - ], - [ - -2.77268729e00, - 1.04680972e-01, - 2.51172238e-02, - 1.95136385e-03, - 3.70052723e-03, - ], - [ - -1.59987770e00, - 2.51172238e-02, - 2.15655720e-02, - 7.65868344e-03, - -7.30173070e-03, - ], - [ - -1.01969471e01, - 1.95136385e-03, - 7.65868344e-03, - 2.78273684e-01, - -6.89402590e-02, - ], - [ - 2.74302006e00, - 3.70052723e-03, - -7.30173070e-03, - -6.89402590e-02, - 7.12034037e-02, - ], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) - hth = np.array( - [ - [ - 4.90000000e01, - 7.04371999e02, - 1.72131237e03, - 7.24743592e02, - 1.70735413e03, - ], - [ - 7.04371999e02, - 1.16866734e04, - 2.15575320e04, - 1.10925200e04, - 2.23848036e04, - ], - [ - 1.72131237e03, - 2.15575320e04, - 7.39058986e04, - 2.34796298e04, - 6.70145378e04, - ], - [ - 7.24743592e02, - 1.10925200e04, - 2.34796298e04, - 1.16146226e04, - 2.30304624e04, - ], - [ - 1.70735413e03, - 2.23848036e04, - 6.70145378e04, - 2.30304624e04, - 6.69879858e04, - ], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - + np.testing.assert_allclose(reg.std_y,stdy,RTOL) + vm = np.array([[ 4.86218274e+02, -2.77268729e+00, -1.59987770e+00, + -1.01969471e+01, 2.74302006e+00], + [ -2.77268729e+00, 1.04680972e-01, 2.51172238e-02, + 1.95136385e-03, 3.70052723e-03], + [ -1.59987770e+00, 2.51172238e-02, 2.15655720e-02, + 7.65868344e-03, -7.30173070e-03], + [ -1.01969471e+01, 1.95136385e-03, 7.65868344e-03, + 2.78273684e-01, -6.89402590e-02], + [ 2.74302006e+00, 3.70052723e-03, -7.30173070e-03, + -6.89402590e-02, 7.12034037e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL*10) + hth = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03, + 7.24743592e+02, 1.70735413e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04, + 1.10925200e+04, 2.23848036e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04, + 2.34796298e+04, 6.70145378e+04], + [ 7.24743592e+02, 1.10925200e+04, 2.34796298e+04, + 1.16146226e+04, 2.30304624e+04], + [ 1.70735413e+03, 2.23848036e+04, 6.70145378e+04, + 2.30304624e+04, 6.69879858e+04]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) class TestGMComboHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): # Only spatial lag reg = HET.GM_Combo_Het(self.y, self.X, w=self.w, step1c=True) - betas = np.array( - [[57.7778574], [0.73034922], [-0.59257362], [-0.2230231], [0.56636724]] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([25.65156033]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([31.87664403]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - ep = np.array([28.30648145]) - np.testing.assert_allclose(reg.e_pred[0], ep, RTOL) - pe = np.array([52.16052155]) - np.testing.assert_allclose(reg.predy_e[0], pe, RTOL) - predy = np.array([54.81544267]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + betas = np.array([[ 57.7778574 ], [ 0.73034922], [ -0.59257362], [ -0.2230231 ], [ 0.56636724]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 25.65156033]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 31.87664403]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + ep = np.array([ 28.30648145]) + np.testing.assert_allclose(reg.e_pred[0],ep,RTOL) + pe = np.array([ 52.16052155]) + np.testing.assert_allclose(reg.predy_e[0],pe,RTOL) + predy = np.array([ 54.81544267]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 4 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0], x, RTOL) - yend = np.array([35.4585005]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([18.594, 24.7142675]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598, 35.4585005]) - np.testing.assert_allclose(reg.z[0], z, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0],x,RTOL) + yend = np.array([ 35.4585005]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 18.594 , 24.7142675]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598 , 35.4585005]) + np.testing.assert_allclose(reg.z[0],z,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [ - 4.86218274e02, - -2.77268729e00, - -1.59987770e00, - -1.01969471e01, - 2.74302006e00, - ], - [ - -2.77268729e00, - 1.04680972e-01, - 2.51172238e-02, - 1.95136385e-03, - 3.70052723e-03, - ], - [ - -1.59987770e00, - 2.51172238e-02, - 2.15655720e-02, - 7.65868344e-03, - -7.30173070e-03, - ], - [ - -1.01969471e01, - 1.95136385e-03, - 7.65868344e-03, - 2.78273684e-01, - -6.89402590e-02, - ], - [ - 2.74302006e00, - 3.70052723e-03, - -7.30173070e-03, - -6.89402590e-02, - 7.12034037e-02, - ], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 4.86218274e+02, -2.77268729e+00, -1.59987770e+00, + -1.01969471e+01, 2.74302006e+00], + [ -2.77268729e+00, 1.04680972e-01, 2.51172238e-02, + 1.95136385e-03, 3.70052723e-03], + [ -1.59987770e+00, 2.51172238e-02, 2.15655720e-02, + 7.65868344e-03, -7.30173070e-03], + [ -1.01969471e+01, 1.95136385e-03, 7.65868344e-03, + 2.78273684e-01, -6.89402590e-02], + [ 2.74302006e+00, 3.70052723e-03, -7.30173070e-03, + -6.89402590e-02, 7.12034037e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL*10) pr2 = 0.3001582877472412 - np.testing.assert_allclose(reg.pr2, pr2, RTOL) + np.testing.assert_allclose(reg.pr2,pr2,RTOL) pr2_e = 0.35613102283621967 - np.testing.assert_allclose(reg.pr2_e, pr2_e, RTOL) - std_err = np.array( - [22.05035768, 0.32354439, 0.14685221, 0.52751653, 0.26683966] - ) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - z_stat = np.array( - [ - (2.6202684885795335, 0.00878605635338265), - (2.2573385444145524, 0.023986928627746887), - (-4.0351698589183433, 5.456281036278686e-05), - (-0.42277935292121521, 0.67245625315942159), - (2.1225002455741895, 0.033795752094112265), - ] - ) - np.testing.assert_allclose(reg.z_stat, z_stat, RTOL) - hth = np.array( - [ - [ - 4.90000000e01, - 7.04371999e02, - 1.72131237e03, - 7.24743592e02, - 1.70735413e03, - ], - [ - 7.04371999e02, - 1.16866734e04, - 2.15575320e04, - 1.10925200e04, - 2.23848036e04, - ], - [ - 1.72131237e03, - 2.15575320e04, - 7.39058986e04, - 2.34796298e04, - 6.70145378e04, - ], - [ - 7.24743592e02, - 1.10925200e04, - 2.34796298e04, - 1.16146226e04, - 2.30304624e04, - ], - [ - 1.70735413e03, - 2.23848036e04, - 6.70145378e04, - 2.30304624e04, - 6.69879858e04, - ], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - + np.testing.assert_allclose(reg.pr2_e,pr2_e,RTOL) + std_err = np.array([ 22.05035768, 0.32354439, 0.14685221, 0.52751653, 0.26683966]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + z_stat = np.array([(2.6202684885795335, 0.00878605635338265), (2.2573385444145524, 0.023986928627746887), (-4.0351698589183433, 5.456281036278686e-05), (-0.42277935292121521, 0.67245625315942159), (2.1225002455741895, 0.033795752094112265)]) + np.testing.assert_allclose(reg.z_stat,z_stat,RTOL) + hth = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03, + 7.24743592e+02, 1.70735413e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04, + 1.10925200e+04, 2.23848036e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04, + 2.34796298e+04, 6.70145378e+04], + [ 7.24743592e+02, 1.10925200e+04, 2.34796298e+04, + 1.16146226e+04, 2.30304624e+04], + [ 1.70735413e+03, 2.23848036e+04, 6.70145378e+04, + 2.30304624e+04, 6.69879858e+04]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/spreg/tests/test_error_sp_het_regimes.py b/spreg/tests/test_error_sp_het_regimes.py index d9070a7e..8606fe8b 100644 --- a/spreg/tests/test_error_sp_het_regimes.py +++ b/spreg/tests/test_error_sp_het_regimes.py @@ -5,13 +5,12 @@ from spreg.error_sp_het import GM_Error_Het, GM_Endog_Error_Het, GM_Combo_Het from libpysal.common import RTOL - class TestGM_Error_Het_Regimes(unittest.TestCase): def setUp(self): - # Columbus: - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + #Columbus: + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("CRIME")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("HOVAL")) X.append(db.by_col("INC")) @@ -25,474 +24,284 @@ def setUp(self): q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Queen.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - self.r_var = "NSA" + self.w = libpysal.weights.Queen.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' + self.r_var = 'NSA' self.regimes = db.by_col(self.r_var) - # Artficial: + #Artficial: n = 256 - self.n2 = n / 2 - self.x_a1 = np.random.uniform(-10, 10, (n, 1)) - self.x_a2 = np.random.uniform(1, 5, (n, 1)) - self.q_a = self.x_a2 + np.random.normal(0, 1, (n, 1)) - self.x_a = np.hstack((self.x_a1, self.x_a2)) - self.y_a = np.dot( - np.hstack((np.ones((n, 1)), self.x_a)), np.array([[1], [0.5], [2]]) - ) + np.random.normal(0, 1, (n, 1)) + self.n2 = n/2 + self.x_a1 = np.random.uniform(-10,10,(n,1)) + self.x_a2 = np.random.uniform(1,5,(n,1)) + self.q_a = self.x_a2 + np.random.normal(0,1,(n,1)) + self.x_a = np.hstack((self.x_a1,self.x_a2)) + self.y_a = np.dot(np.hstack((np.ones((n,1)),self.x_a)),np.array([[1],[0.5],[2]])) + np.random.normal(0,1,(n,1)) latt = int(np.sqrt(n)) - self.w_a = libpysal.weights.util.lat2W(latt, latt) - self.w_a.transform = "r" - self.regi_a = [0] * (n // 2) + [1] * (n // 2) - self.w_a1 = libpysal.weights.util.lat2W(latt // 2, latt) - self.w_a1.transform = "r" - + self.w_a = libpysal.weights.util.lat2W(latt,latt) + self.w_a.transform='r' + self.regi_a = [0]*(n//2) + [1]*(n//2) + self.w_a1 = libpysal.weights.util.lat2W(latt//2,latt) + self.w_a1.transform='r' + def test_model(self): reg = SP.GM_Error_Het_Regimes(self.y, self.X, self.regimes, self.w) - betas = np.array( - [ - [62.95986466], - [-0.15660795], - [-1.49054832], - [60.98577615], - [-0.3358993], - [-0.82129289], - [0.54662719], - ] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) + betas = np.array([[ 62.95986466], + [ -0.15660795], + [ -1.49054832], + [ 60.98577615], + [ -0.3358993 ], + [ -0.82129289], + [ 0.54662719]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) u = np.array([-2.19031456]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([17.91629456]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + np.testing.assert_allclose(reg.u[0],u,RTOL) + predy = np.array([ 17.91629456]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n, RTOL) + np.testing.assert_allclose(reg.n,n,RTOL) k = 6 - np.testing.assert_allclose(reg.k, k, RTOL) - y = np.array([15.72598]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([[0.0, 0.0, 0.0, 1.0, 80.467003, 19.531]]) - np.testing.assert_allclose(reg.x[0].toarray(), x, RTOL) - e = np.array([2.77847355]) - np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) + np.testing.assert_allclose(reg.k,k,RTOL) + y = np.array([ 15.72598]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([[ 0. , 0. , 0. , 1. , 80.467003, 19.531 ]]) + np.testing.assert_allclose(reg.x[0].toarray(),x,RTOL) + e = np.array([ 2.77847355]) + np.testing.assert_allclose(reg.e_filtered[0],e,RTOL) my = 35.128823897959187 - np.testing.assert_allclose(reg.mean_y, my, RTOL) + np.testing.assert_allclose(reg.mean_y,my, RTOL) sy = 16.732092091229699 - np.testing.assert_allclose(reg.std_y, sy, RTOL) - vm = np.array( - [ - 3.86154100e01, - -2.51553730e-01, - -8.20138673e-01, - 1.71714184e00, - -1.94929113e-02, - 1.23118051e-01, - 0.00000000e00, - ] - ) - np.testing.assert_allclose(reg.vm[0], vm, RTOL) + np.testing.assert_allclose(reg.std_y,sy, RTOL) + vm = np.array([ 3.86154100e+01, -2.51553730e-01, -8.20138673e-01, + 1.71714184e+00, -1.94929113e-02, 1.23118051e-01, + 0.00000000e+00]) + np.testing.assert_allclose(reg.vm[0],vm,RTOL*10) pr2 = 0.5515791216043385 - np.testing.assert_allclose(reg.pr2, pr2, RTOL) - std_err = np.array( - [ - 6.21412987, - 0.15340022, - 0.44060473, - 7.6032169, - 0.19353719, - 0.73621596, - 0.13968272, - ] - ) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - chow_r = np.array( - [ - [0.04190799, 0.83779526], - [0.5736724, 0.44880328], - [0.62498575, 0.42920056], - ] - ) - np.testing.assert_allclose(reg.chow.regi, chow_r, RTOL) + np.testing.assert_allclose(reg.pr2,pr2, RTOL) + std_err = np.array([ 6.21412987, 0.15340022, 0.44060473, 7.6032169 , 0.19353719, + 0.73621596, 0.13968272]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + chow_r = np.array([[ 0.04190799, 0.83779526], + [ 0.5736724 , 0.44880328], + [ 0.62498575, 0.42920056]]) + np.testing.assert_allclose(reg.chow.regi,chow_r,RTOL) chow_j = 0.72341901308525713 - np.testing.assert_allclose(reg.chow.joint[0], chow_j, RTOL) + np.testing.assert_allclose(reg.chow.joint[0],chow_j, RTOL) def test_model_regi_error(self): - # Columbus: - reg = SP.GM_Error_Het_Regimes( - self.y, self.X, self.regimes, self.w, regime_err_sep=True - ) - betas = np.array( - [ - [60.74090229], - [-0.17492294], - [-1.33383387], - [0.68303064], - [66.30374279], - [-0.31841139], - [-1.27502813], - [0.11515312], - ] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - vm = np.array([44.9411672, -0.34343354, -0.39946055, 0.0, 0.0, 0.0, 0.0, 0.0]) - np.testing.assert_allclose(reg.vm[0], vm, RTOL) + #Columbus: + reg = SP.GM_Error_Het_Regimes(self.y, self.X, self.regimes, self.w, regime_err_sep=True) + betas = np.array([[ 60.74090229], + [ -0.17492294], + [ -1.33383387], + [ 0.68303064], + [ 66.30374279], + [ -0.31841139], + [ -1.27502813], + [ 0.11515312]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + vm = np.array([ 44.9411672 , -0.34343354, -0.39946055, 0. , + 0. , 0. , 0. , 0. ]) + np.testing.assert_allclose(reg.vm[0],vm,RTOL) u = np.array([-0.05357818]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([15.77955818]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) - e = np.array([0.70542044]) - np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) - chow_r = np.array( - [ - [3.11061225e-01, 5.77029704e-01], - [3.39747489e-01, 5.59975012e-01], - [3.86371771e-03, 9.50436364e-01], - [4.02884201e00, 4.47286322e-02], - ] - ) - np.testing.assert_allclose(reg.chow.regi, chow_r, RTOL) + np.testing.assert_allclose(reg.u[0],u,RTOL) + predy = np.array([ 15.77955818]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) + e = np.array([ 0.70542044]) + np.testing.assert_allclose(reg.e_filtered[0],e,RTOL) + chow_r = np.array([[ 3.11061225e-01, 5.77029704e-01], + [ 3.39747489e-01, 5.59975012e-01], + [ 3.86371771e-03, 9.50436364e-01], + [ 4.02884201e+00, 4.47286322e-02]]) + np.testing.assert_allclose(reg.chow.regi,chow_r,RTOL) chow_j = 4.7467070503995412 - np.testing.assert_allclose(reg.chow.joint[0], chow_j, RTOL) - # Artficial: - model = SP.GM_Error_Het_Regimes( - self.y_a, self.x_a, self.regi_a, w=self.w_a, regime_err_sep=True - ) - model1 = GM_Error_Het( - self.y_a[0 : int(self.n2)].reshape(int(self.n2), 1), - self.x_a[0 : int(self.n2)], - w=self.w_a1, - ) - model2 = GM_Error_Het( - self.y_a[int(self.n2) :].reshape(int(self.n2), 1), - self.x_a[int(self.n2) :], - w=self.w_a1, - ) + np.testing.assert_allclose(reg.chow.joint[0],chow_j, RTOL) + #Artficial: + model = SP.GM_Error_Het_Regimes(self.y_a, self.x_a, self.regi_a, w=self.w_a, regime_err_sep=True) + model1 = GM_Error_Het(self.y_a[0:int(self.n2)].reshape(int(self.n2),1), self.x_a[0:int(self.n2)], w=self.w_a1) + model2 = GM_Error_Het(self.y_a[int(self.n2):].reshape(int(self.n2),1), self.x_a[int(self.n2):], w=self.w_a1) tbetas = np.vstack((model1.betas, model2.betas)) - np.testing.assert_allclose(model.betas, tbetas, RTOL) - vm = np.hstack((model1.vm.diagonal(), model2.vm.diagonal())) + np.testing.assert_allclose(model.betas,tbetas, RTOL) + vm = np.hstack((model1.vm.diagonal(),model2.vm.diagonal())) np.testing.assert_allclose(model.vm.diagonal(), vm, RTOL) def test_model_endog(self): - reg = SP.GM_Endog_Error_Het_Regimes( - self.y, self.X2, self.yd, self.q, self.regimes, self.w - ) - betas = np.array( - [ - [77.26679984], - [4.45992905], - [78.59534391], - [0.41432319], - [-3.20196286], - [-1.13672283], - [0.2174965], - ] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([20.50716917]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - e = np.array([25.13517175]) - np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) + reg = SP.GM_Endog_Error_Het_Regimes(self.y, self.X2, self.yd, self.q, self.regimes, self.w) + betas = np.array([[ 77.26679984], + [ 4.45992905], + [ 78.59534391], + [ 0.41432319], + [ -3.20196286], + [ -1.13672283], + [ 0.2174965 ]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 20.50716917]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + e = np.array([ 25.13517175]) + np.testing.assert_allclose(reg.e_filtered[0],e,RTOL) predy = np.array([-4.78118917]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n, RTOL) + np.testing.assert_allclose(reg.n,n, RTOL) k = 6 - np.testing.assert_allclose(reg.k, k, RTOL) - y = np.array([15.72598]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([[0.0, 0.0, 1.0, 19.531]]) - np.testing.assert_allclose(reg.x[0].toarray(), x, RTOL) - yend = np.array([[0.0, 80.467003]]) - np.testing.assert_allclose(reg.yend[0].toarray(), yend, RTOL) - z = np.array([[0.0, 0.0, 1.0, 19.531, 0.0, 80.467003]]) - np.testing.assert_allclose(reg.z[0].toarray(), z, RTOL) + np.testing.assert_allclose(reg.k,k, RTOL) + y = np.array([ 15.72598]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([[ 0. , 0. , 1. , 19.531]]) + np.testing.assert_allclose(reg.x[0].toarray(),x,RTOL) + yend = np.array([[ 0. , 80.467003]]) + np.testing.assert_allclose(reg.yend[0].toarray(),yend,RTOL) + z = np.array([[ 0. , 0. , 1. , 19.531 , 0. , + 80.467003]]) + np.testing.assert_allclose(reg.z[0].toarray(),z,RTOL) my = 35.128823897959187 - np.testing.assert_allclose(reg.mean_y, my, RTOL) + np.testing.assert_allclose(reg.mean_y,my, RTOL) sy = 16.732092091229699 - np.testing.assert_allclose(reg.std_y, sy, RTOL) - vm = np.array( - [ - 509.66122149, - 150.5845341, - 9.64413821, - 5.54782831, - -80.95846045, - -2.25308524, - -3.2045214, - ] - ) - np.testing.assert_allclose(reg.vm[0], vm, RTOL) + np.testing.assert_allclose(reg.std_y,sy, RTOL) + vm = np.array([ 509.66122149, 150.5845341 , 9.64413821, 5.54782831, + -80.95846045, -2.25308524, -3.2045214 ]) + np.testing.assert_allclose(reg.vm[0],vm,RTOL) pr2 = 0.19776512679331681 - np.testing.assert_allclose(reg.pr2, pr2) - std_err = np.array( - [ - 22.57567765, - 11.34616946, - 17.43881791, - 1.30953812, - 5.4830829, - 0.74634612, - 0.29973079, - ] - ) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - chow_r = np.array( - [[0.0022216, 0.96240654], [0.13127347, 0.7171153], [0.14367307, 0.70465645]] - ) - np.testing.assert_allclose(reg.chow.regi, chow_r, RTOL) + np.testing.assert_allclose(reg.pr2,pr2) + std_err = np.array([ 22.57567765, 11.34616946, 17.43881791, 1.30953812, + 5.4830829 , 0.74634612, 0.29973079]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + chow_r = np.array([[ 0.0022216 , 0.96240654], + [ 0.13127347, 0.7171153 ], + [ 0.14367307, 0.70465645]]) + np.testing.assert_allclose(reg.chow.regi,chow_r,RTOL) chow_j = 1.2329971019087163 - np.testing.assert_allclose(reg.chow.joint[0], chow_j, RTOL) + np.testing.assert_allclose(reg.chow.joint[0],chow_j, RTOL) def test_model_endog_regi_error(self): - # Columbus: - reg = SP.GM_Endog_Error_Het_Regimes( - self.y, self.X2, self.yd, self.q, self.regimes, self.w, regime_err_sep=True - ) - betas = np.array( - [ - [7.92747424e01], - [5.78086230e00], - [-3.83173581e00], - [2.23210962e-01], - [8.26255251e01], - [5.48294187e-01], - [-1.28432891e00], - [3.57661629e-02], - ] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - vm = np.array( - [ - 7.55988579e02, - 2.53659722e02, - -1.34288316e02, - -2.66141766e-01, - 0.00000000e00, - 0.00000000e00, - 0.00000000e00, - 0.00000000e00, - ] - ) - np.testing.assert_allclose(reg.vm[0], vm, RTOL) - u = np.array([25.73781918]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([-10.01183918]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) - e = np.array([26.5449135]) - np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) - chow_r = np.array( - [ - [0.00998573, 0.92040097], - [0.12660165, 0.72198192], - [0.12737281, 0.72117171], - [0.43507956, 0.50950696], - ] - ) - np.testing.assert_allclose(reg.chow.regi, chow_r, RTOL) - chow_j = 1.3756768204399892 - np.testing.assert_allclose(reg.chow.joint[0], chow_j, RTOL) - # Artficial: - model = SP.GM_Endog_Error_Het_Regimes( - self.y_a, - self.x_a1, - yend=self.x_a2, - q=self.q_a, - regimes=self.regi_a, - w=self.w_a, - regime_err_sep=True, - ) - model1 = GM_Endog_Error_Het( - self.y_a[0 : int(self.n2)].reshape(int(self.n2), 1), - self.x_a1[0 : int(self.n2)], - yend=self.x_a2[0 : int(self.n2)], - q=self.q_a[0 : int(self.n2)], - w=self.w_a1, - ) - model2 = GM_Endog_Error_Het( - self.y_a[int(self.n2) :].reshape(int(self.n2), 1), - self.x_a1[int(self.n2) :], - yend=self.x_a2[int(self.n2) :], - q=self.q_a[int(self.n2) :], - w=self.w_a1, - ) + #Columbus: + reg = SP.GM_Endog_Error_Het_Regimes(self.y, self.X2, self.yd, self.q, self.regimes, self.w, regime_err_sep=True) + betas = np.array([[70.45311952], + [ 4.3685181 ], + [-0.9 ], + [ 0.89261347], + [78.43187245], + [ 0.95437565], + [-0.9 ], + [ 0.73629337]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + vm = np.array([ 1.210078e+03, 1.559293e+02, -7.540641e+01, 1.141481e+00, + 0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00]) + np.testing.assert_allclose(reg.vm[0],vm,RTOL) + u = np.array([-8.925501]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + predy = np.array([24.651481]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) + e = np.array([20.195355]) + np.testing.assert_allclose(reg.e_filtered[0],e,RTOL) + chow_r = np.array([[0.041109, 0.839328], + [0.102719, 0.748591], + [0. , 1. ], + [0.311214, 0.576936]]) + np.testing.assert_allclose(reg.chow.regi,chow_r,RTOL) + chow_j = 2.621305 + np.testing.assert_allclose(reg.chow.joint[0],chow_j, RTOL) + #Artficial: + model = SP.GM_Endog_Error_Het_Regimes(self.y_a, self.x_a1, yend=self.x_a2, q=self.q_a, regimes=self.regi_a, w=self.w_a, regime_err_sep=True) + model1 = GM_Endog_Error_Het(self.y_a[0:int(self.n2)].reshape(int(self.n2),1), self.x_a1[0:int(self.n2)], yend=self.x_a2[0:int(self.n2)], q=self.q_a[0:int(self.n2)], w=self.w_a1) + model2 = GM_Endog_Error_Het(self.y_a[int(self.n2):].reshape(int(self.n2),1), self.x_a1[int(self.n2):], yend=self.x_a2[int(self.n2):], q=self.q_a[int(self.n2):], w=self.w_a1) tbetas = np.vstack((model1.betas, model2.betas)) - np.testing.assert_allclose(model.betas, tbetas, RTOL) - vm = np.hstack((model1.vm.diagonal(), model2.vm.diagonal())) + np.testing.assert_allclose(model.betas,tbetas, RTOL) + vm = np.hstack((model1.vm.diagonal(),model2.vm.diagonal())) np.testing.assert_allclose(model.vm.diagonal(), vm, RTOL) def test_model_combo(self): - reg = SP.GM_Combo_Het_Regimes( - self.y, self.X2, self.regimes, self.yd, self.q, w=self.w - ) - betas = np.array( - [ - [3.69372678e01], - [-8.29474998e-01], - [3.08667517e01], - [-7.23753444e-01], - [-3.01900940e-01], - [-2.21328949e-01], - [6.41902155e-01], - [-2.45714919e-02], - ] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([0.94039246]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - e_filtered = np.array([0.8737864]) - np.testing.assert_allclose(reg.e_filtered[0], e_filtered, RTOL) - predy_e = np.array([18.68732105]) - np.testing.assert_allclose(reg.predy_e[0], predy_e, RTOL) - predy = np.array([14.78558754]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + reg = SP.GM_Combo_Het_Regimes(self.y, self.X2, self.regimes, self.yd, self.q, w=self.w) + betas = np.array([[ 3.69372678e+01], + [ -8.29474998e-01], + [ 3.08667517e+01], + [ -7.23753444e-01], + [ -3.01900940e-01], + [ -2.21328949e-01], + [ 6.41902155e-01], + [ -2.45714919e-02]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 0.94039246]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + e_filtered = np.array([ 0.8737864]) + np.testing.assert_allclose(reg.e_filtered[0],e_filtered,RTOL) + predy_e = np.array([ 18.68732105]) + np.testing.assert_allclose(reg.predy_e[0],predy_e,RTOL) + predy = np.array([ 14.78558754]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n, RTOL) + np.testing.assert_allclose(reg.n,n, RTOL) k = 7 - np.testing.assert_allclose(reg.k, k, RTOL) - y = np.array([15.72598]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([[0.0, 0.0, 1.0, 19.531]]) - np.testing.assert_allclose(reg.x[0].toarray(), x, RTOL) - yend = np.array([[0.0, 80.467003, 24.7142675]]) - np.testing.assert_allclose(reg.yend[0].toarray(), yend, RTOL) - z = np.array([[0.0, 0.0, 1.0, 19.531, 0.0, 80.467003, 24.7142675]]) - np.testing.assert_allclose(reg.z[0].toarray(), z, RTOL) + np.testing.assert_allclose(reg.k,k, RTOL) + y = np.array([ 15.72598]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([[ 0. , 0. , 1. , 19.531]]) + np.testing.assert_allclose(reg.x[0].toarray(),x,RTOL) + yend = np.array([[ 0. , 80.467003 , 24.7142675]]) + np.testing.assert_allclose(reg.yend[0].toarray(),yend,RTOL) + z = np.array([[ 0. , 0. , 1. , 19.531 , 0. , + 80.467003 , 24.7142675]]) + np.testing.assert_allclose(reg.z[0].toarray(),z,RTOL) my = 35.128823897959187 - np.testing.assert_allclose(reg.mean_y, my, RTOL) + np.testing.assert_allclose(reg.mean_y,my, RTOL) sy = 16.732092091229699 - np.testing.assert_allclose(reg.std_y, sy, RTOL) - vm = np.array( - [ - 71.26851365, - -0.58278032, - 50.53169815, - -0.74561147, - -0.79510274, - -0.10823496, - -0.98141395, - 1.16575965, - ] - ) - np.testing.assert_allclose(reg.vm[0], vm, RTOL) + np.testing.assert_allclose(reg.std_y,sy, RTOL) + vm = np.array([ 71.26851365, -0.58278032, 50.53169815, -0.74561147, + -0.79510274, -0.10823496, -0.98141395, 1.16575965]) + np.testing.assert_allclose(reg.vm[0],vm,RTOL) pr2 = 0.6504148883602958 - np.testing.assert_allclose(reg.pr2, pr2, RTOL) + np.testing.assert_allclose(reg.pr2,pr2,RTOL) pr2_e = 0.527136896994038 - np.testing.assert_allclose(reg.pr2_e, pr2_e, RTOL) - std_err = np.array( - [ - 8.44206809, - 0.72363219, - 9.85790968, - 0.77218082, - 0.34084146, - 0.21752916, - 0.14371614, - 0.39226478, - ] - ) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - chow_r = np.array( - [ - [0.54688708, 0.45959243], - [0.01035136, 0.91896175], - [0.03981108, 0.84185042], - ] - ) - np.testing.assert_allclose(reg.chow.regi, chow_r, RTOL) + np.testing.assert_allclose(reg.pr2_e,pr2_e,RTOL) + std_err = np.array([ 8.44206809, 0.72363219, 9.85790968, 0.77218082, 0.34084146, + 0.21752916, 0.14371614, 0.39226478]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + chow_r = np.array([[ 0.54688708, 0.45959243], + [ 0.01035136, 0.91896175], + [ 0.03981108, 0.84185042]]) + np.testing.assert_allclose(reg.chow.regi,chow_r,RTOL) chow_j = 0.78070369988354349 - np.testing.assert_allclose(reg.chow.joint[0], chow_j, RTOL) + np.testing.assert_allclose(reg.chow.joint[0],chow_j,RTOL) def test_model_combo_regi_error(self): - # Columbus: - reg = SP.GM_Combo_Het_Regimes( - self.y, - self.X2, - self.regimes, - self.yd, - self.q, - w=self.w, - regime_lag_sep=True, - regime_err_sep=True, - ) - betas = np.array( - [ - [42.01151458], - [-0.13917151], - [-0.65300184], - [0.54737064], - [0.2629229], - [34.21569751], - [-0.15236089], - [-0.49175217], - [0.65733173], - [-0.07713581], - ] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - vm = np.array( - [ - 77.49519689, - 0.57226879, - -1.18856422, - -1.28088712, - 0.866752, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - ] - ) - np.testing.assert_allclose(reg.vm[0], vm, RTOL) - u = np.array([7.81039418]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([7.91558582]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) - e = np.array([7.22996911]) - np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) - chow_r = np.array( - [ - [1.90869079e-01, 6.62194273e-01], - [4.56118982e-05, 9.94611401e-01], - [3.12104263e-02, 8.59771748e-01], - [1.56368204e-01, 6.92522476e-01], - [7.52928732e-01, 3.85550558e-01], - ] - ) - np.testing.assert_allclose(reg.chow.regi, chow_r, RTOL) + #Columbus: + reg = SP.GM_Combo_Het_Regimes(self.y, self.X2, self.regimes, self.yd, self.q, w=self.w, regime_lag_sep=True, regime_err_sep=True) + betas = np.array([[ 42.01151458], + [ -0.13917151], + [ -0.65300184], + [ 0.54737064], + [ 0.2629229 ], + [ 34.21569751], + [ -0.15236089], + [ -0.49175217], + [ 0.65733173], + [ -0.07713581]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + vm = np.array([ 77.49519689, 0.57226879, -1.18856422, -1.28088712, + 0.866752 , 0. , 0. , 0. , + 0. , 0. ]) + np.testing.assert_allclose(reg.vm[0],vm,RTOL) + u = np.array([ 7.81039418]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + predy = np.array([ 7.91558582]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) + e = np.array([ 7.22996911]) + np.testing.assert_allclose(reg.e_filtered[0],e,RTOL) + chow_r = np.array([[ 1.90869079e-01, 6.62194273e-01], + [ 4.56118982e-05, 9.94611401e-01], + [ 3.12104263e-02, 8.59771748e-01], + [ 1.56368204e-01, 6.92522476e-01], + [ 7.52928732e-01, 3.85550558e-01]]) + np.testing.assert_allclose(reg.chow.regi,chow_r,RTOL) chow_j = 1.1316136604755913 - np.testing.assert_allclose(reg.chow.joint[0], chow_j, RTOL) - # Artficial: - model = SP.GM_Combo_Het_Regimes( - self.y_a, - self.x_a1, - yend=self.x_a2, - q=self.q_a, - regimes=self.regi_a, - w=self.w_a, - regime_err_sep=True, - regime_lag_sep=True, - ) - model1 = GM_Combo_Het( - self.y_a[0 : int(self.n2)].reshape(int(self.n2), 1), - self.x_a1[0 : int(self.n2)], - yend=self.x_a2[0 : int(self.n2)], - q=self.q_a[0 : int(self.n2)], - w=self.w_a1, - ) - model2 = GM_Combo_Het( - self.y_a[int(self.n2) :].reshape(int(self.n2), 1), - self.x_a1[int(self.n2) :], - yend=self.x_a2[int(self.n2) :], - q=self.q_a[int(self.n2) :], - w=self.w_a1, - ) + np.testing.assert_allclose(reg.chow.joint[0],chow_j,RTOL) + #Artficial: + model = SP.GM_Combo_Het_Regimes(self.y_a, self.x_a1, yend=self.x_a2, q=self.q_a, regimes=self.regi_a, w=self.w_a, regime_err_sep=True, regime_lag_sep=True) + model1 = GM_Combo_Het(self.y_a[0:int(self.n2)].reshape(int(self.n2),1), self.x_a1[0:int(self.n2)], yend=self.x_a2[0:int(self.n2)], q=self.q_a[0:int(self.n2)], w=self.w_a1) + model2 = GM_Combo_Het(self.y_a[int(self.n2):].reshape(int(self.n2),1), self.x_a1[int(self.n2):], yend=self.x_a2[int(self.n2):], q=self.q_a[int(self.n2):], w=self.w_a1) tbetas = np.vstack((model1.betas, model2.betas)) - np.testing.assert_allclose(model.betas, tbetas, RTOL) - vm = np.hstack((model1.vm.diagonal(), model2.vm.diagonal())) - # was this supposed to get tested? - + np.testing.assert_allclose(model.betas,tbetas,RTOL) + vm = np.hstack((model1.vm.diagonal(),model2.vm.diagonal())) + #was this supposed to get tested? -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/spreg/tests/test_error_sp_het_sparse.py b/spreg/tests/test_error_sp_het_sparse.py index 16b30328..2890d454 100644 --- a/spreg/tests/test_error_sp_het_sparse.py +++ b/spreg/tests/test_error_sp_het_sparse.py @@ -6,150 +6,131 @@ from libpysal.common import RTOL import spreg - class TestBaseGMErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) self.X = sparse.csr_matrix(self.X) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): reg = HET.BaseGM_Error_Het(self.y, self.X, self.w.sparse, step1c=True) - betas = np.array([[47.99626638], [0.71048989], [-0.55876126], [0.41178776]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([27.38122697]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([32.29765975]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([53.08577603]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + betas = np.array([[ 47.99626638], [ 0.71048989], [ -0.55876126], [ 0.41178776]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 27.38122697]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 32.29765975]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 53.08577603]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0].toarray()[0], x, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0].toarray()[0],x,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [1.31767529e02, -3.58368748e00, -1.65090647e00, 0.00000000e00], - [-3.58368748e00, 1.35513711e-01, 3.77539055e-02, 0.00000000e00], - [-1.65090647e00, 3.77539055e-02, 2.61042702e-02, 0.00000000e00], - [0.00000000e00, 0.00000000e00, 0.00000000e00, 2.82398517e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) - xtx = np.array( - [ - [4.90000000e01, 7.04371999e02, 1.72131237e03], - [7.04371999e02, 1.16866734e04, 2.15575320e04], - [1.72131237e03, 2.15575320e04, 7.39058986e04], - ] - ) - np.testing.assert_allclose(reg.xtx, xtx, RTOL) - - + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 1.31767529e+02, -3.58368748e+00, -1.65090647e+00, + 0.00000000e+00], + [ -3.58368748e+00, 1.35513711e-01, 3.77539055e-02, + 0.00000000e+00], + [ -1.65090647e+00, 3.77539055e-02, 2.61042702e-02, + 0.00000000e+00], + [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, + 2.82398517e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) + xtx = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04]]) + np.testing.assert_allclose(reg.xtx,xtx,RTOL) + class TestGMErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T self.X = sparse.csr_matrix(self.X) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): reg = HET.GM_Error_Het(self.y, self.X, self.w, step1c=True) - betas = np.array([[47.99626638], [0.71048989], [-0.55876126], [0.41178776]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([27.38122697]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([32.29765975]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([53.08577603]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + betas = np.array([[ 47.99626638], [ 0.71048989], [ -0.55876126], [ 0.41178776]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 27.38122697]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 32.29765975]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 53.08577603]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0].toarray()[0], x, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0].toarray()[0],x,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [1.31767529e02, -3.58368748e00, -1.65090647e00, 0.00000000e00], - [-3.58368748e00, 1.35513711e-01, 3.77539055e-02, 0.00000000e00], - [-1.65090647e00, 3.77539055e-02, 2.61042702e-02, 0.00000000e00], - [0.00000000e00, 0.00000000e00, 0.00000000e00, 2.82398517e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 1.31767529e+02, -3.58368748e+00, -1.65090647e+00, + 0.00000000e+00], + [ -3.58368748e+00, 1.35513711e-01, 3.77539055e-02, + 0.00000000e+00], + [ -1.65090647e+00, 3.77539055e-02, 2.61042702e-02, + 0.00000000e+00], + [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, + 2.82398517e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) pr2 = 0.34951013222581306 - np.testing.assert_allclose(reg.pr2, pr2) - stde = np.array([11.47900385, 0.36812187, 0.16156816, 0.16804717]) - np.testing.assert_allclose(reg.std_err, stde, RTOL) - z_stat = np.array( - [ - [4.18122226e00, 2.89946274e-05], - [1.93003988e00, 5.36018970e-02], - [-3.45836247e00, 5.43469673e-04], - [2.45042960e00, 1.42685863e-02], - ] - ) - np.testing.assert_allclose(reg.z_stat, z_stat, RTOL) - xtx = np.array( - [ - [4.90000000e01, 7.04371999e02, 1.72131237e03], - [7.04371999e02, 1.16866734e04, 2.15575320e04], - [1.72131237e03, 2.15575320e04, 7.39058986e04], - ] - ) - np.testing.assert_allclose(reg.xtx, xtx, RTOL) - + np.testing.assert_allclose(reg.pr2,pr2) + stde = np.array([ 11.47900385, 0.36812187, 0.16156816, 0.16804717]) + np.testing.assert_allclose(reg.std_err,stde,RTOL) + z_stat = np.array([[ 4.18122226e+00, 2.89946274e-05], + [ 1.93003988e+00, 5.36018970e-02], + [ -3.45836247e+00, 5.43469673e-04], + [ 2.45042960e+00, 1.42685863e-02]]) + np.testing.assert_allclose(reg.z_stat,z_stat,RTOL) + xtx = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04]]) + np.testing.assert_allclose(reg.xtx,xtx,RTOL) class TestBaseGMEndogErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) self.X = sparse.csr_matrix(self.X) yd = [] yd.append(db.by_col("CRIME")) @@ -157,71 +138,62 @@ def setUp(self): q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HET.BaseGM_Endog_Error_Het( - self.y, self.X, self.yd, self.q, self.w.sparse, step1c=True - ) - betas = np.array([[55.39707924], [0.46563046], [-0.67038326], [0.41135023]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([26.51812895]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([31.46604707]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([53.94887405]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + reg = HET.BaseGM_Endog_Error_Het(self.y, self.X, self.yd, self.q, self.w.sparse, step1c=True) + betas = np.array([[ 55.39707924], [ 0.46563046], [ -0.67038326], [ 0.41135023]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 26.51812895]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 31.46604707]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 53.94887405]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531]) - np.testing.assert_allclose(reg.x[0].toarray()[0], x, RTOL) - yend = np.array([15.72598]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([5.03]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.z[0].toarray()[0], z, RTOL) - h = np.array([1.0, 19.531, 5.03]) - np.testing.assert_allclose(reg.h[0].toarray()[0], h, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531]) + np.testing.assert_allclose(reg.x[0].toarray()[0],x,RTOL) + yend = np.array([ 15.72598]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 5.03]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.z[0].toarray()[0],z,RTOL) + h = np.array([ 1. , 19.531, 5.03 ]) + np.testing.assert_allclose(reg.h[0].toarray()[0],h,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [8.34637805e02, -2.16932259e01, -1.33327894e01, 1.65840848e00], - [-2.16932259e01, 5.97683070e-01, 3.39503523e-01, -3.90111107e-02], - [-1.33327894e01, 3.39503523e-01, 2.19008080e-01, -2.81929695e-02], - [1.65840848e00, -3.90111107e-02, -2.81929695e-02, 3.15686105e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) - hth = np.array( - [ - [49.0, 704.371999, 139.75], - [704.371999, 11686.67338121, 2246.12800625], - [139.75, 2246.12800625, 498.5851], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - - + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 8.34637805e+02, -2.16932259e+01, -1.33327894e+01, + 1.65840848e+00], + [ -2.16932259e+01, 5.97683070e-01, 3.39503523e-01, + -3.90111107e-02], + [ -1.33327894e+01, 3.39503523e-01, 2.19008080e-01, + -2.81929695e-02], + [ 1.65840848e+00, -3.90111107e-02, -2.81929695e-02, + 3.15686105e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) + hth = np.array([[ 49. , 704.371999 , 139.75 ], + [ 704.371999 , 11686.67338121, 2246.12800625], + [ 139.75 , 2246.12800625, 498.5851 ]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) + class TestGMEndogErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T @@ -232,366 +204,213 @@ def setUp(self): q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HET.GM_Endog_Error_Het( - self.y, self.X, self.yd, self.q, self.w, step1c=True - ) - betas = np.array([[55.39707924], [0.46563046], [-0.67038326], [0.41135023]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([26.51812895]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([53.94887405]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + reg = HET.GM_Endog_Error_Het(self.y, self.X, self.yd, self.q, self.w, step1c=True) + betas = np.array([[ 55.39707924], [ 0.46563046], [ -0.67038326], [ 0.41135023]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 26.51812895]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + predy = np.array([ 53.94887405]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531]) - np.testing.assert_allclose(reg.x[0].toarray()[0], x, RTOL) - yend = np.array([15.72598]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([5.03]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.z[0].toarray()[0], z, RTOL) - h = np.array([1.0, 19.531, 5.03]) - np.testing.assert_allclose(reg.h[0].toarray()[0], h, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531]) + np.testing.assert_allclose(reg.x[0].toarray()[0],x,RTOL) + yend = np.array([ 15.72598]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 5.03]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.z[0].toarray()[0],z,RTOL) + h = np.array([ 1. , 19.531, 5.03 ]) + np.testing.assert_allclose(reg.h[0].toarray()[0],h,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [8.34637805e02, -2.16932259e01, -1.33327894e01, 1.65840848e00], - [-2.16932259e01, 5.97683070e-01, 3.39503523e-01, -3.90111107e-02], - [-1.33327894e01, 3.39503523e-01, 2.19008080e-01, -2.81929695e-02], - [1.65840848e00, -3.90111107e-02, -2.81929695e-02, 3.15686105e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 8.34637805e+02, -2.16932259e+01, -1.33327894e+01, + 1.65840848e+00], + [ -2.16932259e+01, 5.97683070e-01, 3.39503523e-01, + -3.90111107e-02], + [ -1.33327894e+01, 3.39503523e-01, 2.19008080e-01, + -2.81929695e-02], + [ 1.65840848e+00, -3.90111107e-02, -2.81929695e-02, + 3.15686105e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) pr2 = 0.34648011338954804 - np.testing.assert_allclose(reg.pr2, pr2, RTOL) - std_err = np.array([28.89009873, 0.77309965, 0.46798299, 0.17767558]) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - z_stat = np.array( - [ - (1.9175109006819244, 0.055173057472126787), - (0.60229035155742305, 0.54698088217644414), - (-1.4324949211864271, 0.15200223057569454), - (2.3151759776869496, 0.020603303355572443), - ] - ) - np.testing.assert_allclose(reg.z_stat, z_stat, RTOL) - hth = np.array( - [ - [49.0, 704.371999, 139.75], - [704.371999, 11686.67338121, 2246.12800625], - [139.75, 2246.12800625, 498.5851], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - - + np.testing.assert_allclose(reg.pr2,pr2,RTOL) + std_err = np.array([ 28.89009873, 0.77309965, 0.46798299, + 0.17767558]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + z_stat = np.array([(1.9175109006819244, 0.055173057472126787), (0.60229035155742305, 0.54698088217644414), (-1.4324949211864271, 0.15200223057569454), (2.3151759776869496, 0.020603303355572443)]) + np.testing.assert_allclose(reg.z_stat,z_stat,RTOL) + hth = np.array([[ 49. , 704.371999 , 139.75 ], + [ 704.371999 , 11686.67338121, 2246.12800625], + [ 139.75 , 2246.12800625, 498.5851 ]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) + class TestBaseGMComboHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): # Only spatial lag yd2, q2 = spreg.utils.set_endog(self.y, self.X, self.w, None, None, 1, True) - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) self.X = sparse.csr_matrix(self.X) - reg = HET.BaseGM_Combo_Het( - self.y, self.X, yend=yd2, q=q2, w=self.w.sparse, step1c=True - ) - betas = np.array( - [[57.7778574], [0.73034922], [-0.59257362], [-0.2230231], [0.56636724]] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([25.65156033]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([31.87664403]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([54.81544267]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + reg = HET.BaseGM_Combo_Het(self.y, self.X, yend=yd2, q=q2, w=self.w.sparse, step1c=True) + betas = np.array([[ 57.7778574 ], [ 0.73034922], [ -0.59257362], [ -0.2230231 ], [ 0.56636724]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 25.65156033]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 31.87664403]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 54.81544267]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 4 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0].toarray()[0], x, RTOL) - yend = np.array([35.4585005]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([18.594, 24.7142675]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598, 35.4585005]) - np.testing.assert_allclose(reg.z[0].toarray()[0], z, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0].toarray()[0],x,RTOL) + yend = np.array([ 35.4585005]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 18.594 , 24.7142675]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598 , 35.4585005]) + np.testing.assert_allclose(reg.z[0].toarray()[0],z,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy, RTOL) - vm = np.array( - [ - [ - 4.86218274e02, - -2.77268729e00, - -1.59987770e00, - -1.01969471e01, - 2.74302006e00, - ], - [ - -2.77268729e00, - 1.04680972e-01, - 2.51172238e-02, - 1.95136385e-03, - 3.70052723e-03, - ], - [ - -1.59987770e00, - 2.51172238e-02, - 2.15655720e-02, - 7.65868344e-03, - -7.30173070e-03, - ], - [ - -1.01969471e01, - 1.95136385e-03, - 7.65868344e-03, - 2.78273684e-01, - -6.89402590e-02, - ], - [ - 2.74302006e00, - 3.70052723e-03, - -7.30173070e-03, - -6.89402590e-02, - 7.12034037e-02, - ], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) - hth = np.array( - [ - [ - 4.90000000e01, - 7.04371999e02, - 1.72131237e03, - 7.24743592e02, - 1.70735413e03, - ], - [ - 7.04371999e02, - 1.16866734e04, - 2.15575320e04, - 1.10925200e04, - 2.23848036e04, - ], - [ - 1.72131237e03, - 2.15575320e04, - 7.39058986e04, - 2.34796298e04, - 6.70145378e04, - ], - [ - 7.24743592e02, - 1.10925200e04, - 2.34796298e04, - 1.16146226e04, - 2.30304624e04, - ], - [ - 1.70735413e03, - 2.23848036e04, - 6.70145378e04, - 2.30304624e04, - 6.69879858e04, - ], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - + np.testing.assert_allclose(reg.std_y,stdy,RTOL) + vm = np.array([[ 4.86218274e+02, -2.77268729e+00, -1.59987770e+00, + -1.01969471e+01, 2.74302006e+00], + [ -2.77268729e+00, 1.04680972e-01, 2.51172238e-02, + 1.95136385e-03, 3.70052723e-03], + [ -1.59987770e+00, 2.51172238e-02, 2.15655720e-02, + 7.65868344e-03, -7.30173070e-03], + [ -1.01969471e+01, 1.95136385e-03, 7.65868344e-03, + 2.78273684e-01, -6.89402590e-02], + [ 2.74302006e+00, 3.70052723e-03, -7.30173070e-03, + -6.89402590e-02, 7.12034037e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL*10) + hth = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03, + 7.24743592e+02, 1.70735413e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04, + 1.10925200e+04, 2.23848036e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04, + 2.34796298e+04, 6.70145378e+04], + [ 7.24743592e+02, 1.10925200e+04, 2.34796298e+04, + 1.16146226e+04, 2.30304624e+04], + [ 1.70735413e+03, 2.23848036e+04, 6.70145378e+04, + 2.30304624e+04, 6.69879858e+04]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) class TestGMComboHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T self.X = sparse.csr_matrix(self.X) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): # Only spatial lag reg = HET.GM_Combo_Het(self.y, self.X, w=self.w, step1c=True) - betas = np.array( - [[57.7778574], [0.73034922], [-0.59257362], [-0.2230231], [0.56636724]] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([25.65156033]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([31.87664403]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - ep = np.array([28.30648145]) - np.testing.assert_allclose(reg.e_pred[0], ep, RTOL) - pe = np.array([52.16052155]) - np.testing.assert_allclose(reg.predy_e[0], pe, RTOL) - predy = np.array([54.81544267]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + betas = np.array([[ 57.7778574 ], [ 0.73034922], [ -0.59257362], [ -0.2230231 ], [ 0.56636724]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 25.65156033]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 31.87664403]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + ep = np.array([ 28.30648145]) + np.testing.assert_allclose(reg.e_pred[0],ep,RTOL) + pe = np.array([ 52.16052155]) + np.testing.assert_allclose(reg.predy_e[0],pe,RTOL) + predy = np.array([ 54.81544267]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 4 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0].toarray()[0], x, RTOL) - yend = np.array([35.4585005]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([18.594, 24.7142675]) - np.testing.assert_allclose(reg.q[0].toarray()[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598, 35.4585005]) - np.testing.assert_allclose(reg.z[0].toarray()[0], z, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0].toarray()[0],x,RTOL) + yend = np.array([ 35.4585005]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 18.594 , 24.7142675]) + np.testing.assert_allclose(reg.q[0].toarray()[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598 , 35.4585005]) + np.testing.assert_allclose(reg.z[0].toarray()[0],z,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [ - 4.86218274e02, - -2.77268729e00, - -1.59987770e00, - -1.01969471e01, - 2.74302006e00, - ], - [ - -2.77268729e00, - 1.04680972e-01, - 2.51172238e-02, - 1.95136385e-03, - 3.70052723e-03, - ], - [ - -1.59987770e00, - 2.51172238e-02, - 2.15655720e-02, - 7.65868344e-03, - -7.30173070e-03, - ], - [ - -1.01969471e01, - 1.95136385e-03, - 7.65868344e-03, - 2.78273684e-01, - -6.89402590e-02, - ], - [ - 2.74302006e00, - 3.70052723e-03, - -7.30173070e-03, - -6.89402590e-02, - 7.12034037e-02, - ], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 4.86218274e+02, -2.77268729e+00, -1.59987770e+00, + -1.01969471e+01, 2.74302006e+00], + [ -2.77268729e+00, 1.04680972e-01, 2.51172238e-02, + 1.95136385e-03, 3.70052723e-03], + [ -1.59987770e+00, 2.51172238e-02, 2.15655720e-02, + 7.65868344e-03, -7.30173070e-03], + [ -1.01969471e+01, 1.95136385e-03, 7.65868344e-03, + 2.78273684e-01, -6.89402590e-02], + [ 2.74302006e+00, 3.70052723e-03, -7.30173070e-03, + -6.89402590e-02, 7.12034037e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL*10) pr2 = 0.3001582877472412 - np.testing.assert_allclose(reg.pr2, pr2, RTOL) + np.testing.assert_allclose(reg.pr2,pr2,RTOL) pr2_e = 0.35613102283621967 - np.testing.assert_allclose(reg.pr2_e, pr2_e, RTOL) - std_err = np.array( - [22.05035768, 0.32354439, 0.14685221, 0.52751653, 0.26683966] - ) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - z_stat = np.array( - [ - (2.6202684885795335, 0.00878605635338265), - (2.2573385444145524, 0.023986928627746887), - (-4.0351698589183433, 5.456281036278686e-05), - (-0.42277935292121521, 0.67245625315942159), - (2.1225002455741895, 0.033795752094112265), - ] - ) - np.testing.assert_allclose(reg.z_stat, z_stat, RTOL) - hth = np.array( - [ - [ - 4.90000000e01, - 7.04371999e02, - 1.72131237e03, - 7.24743592e02, - 1.70735413e03, - ], - [ - 7.04371999e02, - 1.16866734e04, - 2.15575320e04, - 1.10925200e04, - 2.23848036e04, - ], - [ - 1.72131237e03, - 2.15575320e04, - 7.39058986e04, - 2.34796298e04, - 6.70145378e04, - ], - [ - 7.24743592e02, - 1.10925200e04, - 2.34796298e04, - 1.16146226e04, - 2.30304624e04, - ], - [ - 1.70735413e03, - 2.23848036e04, - 6.70145378e04, - 2.30304624e04, - 6.69879858e04, - ], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - + np.testing.assert_allclose(reg.pr2_e,pr2_e,RTOL) + std_err = np.array([ 22.05035768, 0.32354439, 0.14685221, 0.52751653, 0.26683966]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + z_stat = np.array([(2.6202684885795335, 0.00878605635338265), (2.2573385444145524, 0.023986928627746887), (-4.0351698589183433, 5.456281036278686e-05), (-0.42277935292121521, 0.67245625315942159), (2.1225002455741895, 0.033795752094112265)]) + np.testing.assert_allclose(reg.z_stat,z_stat,RTOL) + hth = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03, + 7.24743592e+02, 1.70735413e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04, + 1.10925200e+04, 2.23848036e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04, + 2.34796298e+04, 6.70145378e+04], + [ 7.24743592e+02, 1.10925200e+04, 2.34796298e+04, + 1.16146226e+04, 2.30304624e+04], + [ 1.70735413e+03, 2.23848036e+04, 6.70145378e+04, + 2.30304624e+04, 6.69879858e+04]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/spreg/tests/test_sp_panels.py b/spreg/tests/test_sp_panels.py index b940c5c1..4710d1e8 100755 --- a/spreg/tests/test_sp_panels.py +++ b/spreg/tests/test_sp_panels.py @@ -3,192 +3,87 @@ import libpysal from libpysal.common import RTOL from spreg.sp_panels import * - ATOL = 1e-12 class Test_GM_KKP(unittest.TestCase): def setUp(self): - nat = libpysal.examples.load_example("NCOVR") - self.db = libpysal.io.open(nat.get_path("NAT.dbf"), "r") - self.w = libpysal.weights.Queen.from_shapefile( - libpysal.examples.get_path("NAT.shp") - ) - self.w.transform = "r" - self.y_var0 = ["HR70", "HR80", "HR90"] - self.x_var0 = ["RD70", "RD80", "RD90", "PS70", "PS80", "PS90"] + nat = libpysal.examples.load_example('NCOVR') + self.db = libpysal.io.open(nat.get_path("NAT.dbf"),'r') + self.w = libpysal.weights.Queen.from_shapefile(libpysal.examples.get_path("NAT.shp")) + self.w.transform = 'r' + self.y_var0 = ['HR70','HR80','HR90'] + self.x_var0 = ['RD70','RD80','RD90','PS70','PS80','PS90'] self.y = np.array([self.db.by_col(name) for name in self.y_var0]).T self.x = np.array([self.db.by_col(name) for name in self.x_var0]).T - def test_wide_ident(self): - reg = GM_KKP( - self.y, - self.x, - self.w, - full_weights=False, - name_y=self.y_var0, - name_x=self.x_var0, - ) - np.testing.assert_allclose( - reg.betas, - np.array( - [ - [6.49221562], - [3.62445753], - [1.31187779], - [0.41777589], - [22.81908224], - [39.90993228], - ] - ), - RTOL, - ) - np.testing.assert_allclose( - reg.vm, - np.array( - [ - [1.26948117e-02, -1.98160325e-06, 7.38157674e-05], - [-1.98160325e-06, 7.69961725e-03, 1.13099329e-03], - [7.38157674e-05, 1.13099329e-03, 7.26783636e-03], - ] - ), - RTOL, - ) - np.testing.assert_equal( - reg.name_x, ["CONSTANT", "RD", "PS", "lambda", " sigma2_v", "sigma2_1"] - ) - np.testing.assert_equal(reg.name_y, "HR") - def test_wide_full(self): - reg = GM_KKP(self.y, self.x, self.w, full_weights=True) + def test_wide_ident(self): + reg = GM_KKP(self.y,self.x,self.w,full_weights=False,name_y=self.y_var0, name_x=self.x_var0) + np.testing.assert_allclose(reg.betas,np.array([[ 6.49221562], + [ 3.62445753], + [ 1.31187779], + [ 0.41777589], + [22.81908224], + [39.90993228]]),RTOL) + np.testing.assert_allclose(reg.vm,np.array([[ 1.26948117e-02, -1.98160325e-06, 7.38157674e-05], + [-1.98160325e-06, 7.69961725e-03, 1.13099329e-03], + [ 7.38157674e-05, 1.13099329e-03, 7.26783636e-03]]),RTOL*10) + np.testing.assert_equal(reg.name_x, ['CONSTANT', 'RD', 'PS', 'lambda', ' sigma2_v', 'sigma2_1']) + np.testing.assert_equal(reg.name_y, 'HR') + + def test_wide_full(self): + reg = GM_KKP(self.y,self.x,self.w,full_weights=True) - np.testing.assert_allclose( - reg.betas, - np.array( - [ - [6.49193589], - [3.55740165], - [1.29462748], - [0.4263399], - [22.47241979], - [45.82593532], - ] - ), - RTOL, - ) - np.testing.assert_allclose( - reg.vm, - np.array( - [ - [1.45113773e-02, -2.14882672e-06, 8.54997693e-05], - [-2.14882672e-06, 8.41929187e-03, 1.24553497e-03], - [8.54997693e-05, 1.24553497e-03, 8.12448812e-03], - ] - ), - RTOL, - ) + np.testing.assert_allclose(reg.betas,np.array([[ 6.49193589], + [ 3.55740165], + [ 1.29462748], + [ 0.4263399 ], + [22.47241979], + [45.82593532]]),RTOL) + np.testing.assert_allclose(reg.vm,np.array([[ 1.45113773e-02, -2.14882672e-06, 8.54997693e-05], + [-2.14882672e-06, 8.41929187e-03, 1.24553497e-03], + [ 8.54997693e-05, 1.24553497e-03, 8.12448812e-03]]),RTOL) - def test_long_ident(self): - bigy = self.y.reshape((self.y.size, 1), order="F") - bigx = self.x[:, 0:3].reshape((self.x.shape[0] * 3, 1), order="F") - bigx = np.hstack( - (bigx, self.x[:, 3:6].reshape((self.x.shape[0] * 3, 1), order="F")) - ) - reg = GM_KKP( - bigy, bigx, self.w, full_weights=False, name_y=["HR"], name_x=["RD", "PS"] - ) + def test_long_ident(self): + bigy = self.y.reshape((self.y.size,1),order="F") + bigx = self.x[:,0:3].reshape((self.x.shape[0]*3,1),order='F') + bigx = np.hstack((bigx,self.x[:,3:6].reshape((self.x.shape[0]*3,1),order='F'))) + reg = GM_KKP(bigy,bigx,self.w,full_weights=False,name_y=['HR'], name_x=['RD','PS']) - np.testing.assert_allclose( - reg.betas, - np.array( - [ - [6.49221562], - [3.62445753], - [1.31187779], - [0.41777589], - [22.81908224], - [39.90993228], - ] - ), - RTOL, - ) - np.testing.assert_allclose( - reg.vm, - np.array( - [ - [1.26948117e-02, -1.98160325e-06, 7.38157674e-05], - [-1.98160325e-06, 7.69961725e-03, 1.13099329e-03], - [7.38157674e-05, 1.13099329e-03, 7.26783636e-03], - ] - ), - RTOL, - ) - np.testing.assert_equal( - reg.name_x, ["CONSTANT", "RD", "PS", "lambda", " sigma2_v", "sigma2_1"] - ) - np.testing.assert_equal(reg.name_y, "HR") + np.testing.assert_allclose(reg.betas,np.array([[ 6.49221562], + [ 3.62445753], + [ 1.31187779], + [ 0.41777589], + [22.81908224], + [39.90993228]]),RTOL) + np.testing.assert_allclose(reg.vm,np.array([[ 1.26948117e-02, -1.98160325e-06, 7.38157674e-05], + [-1.98160325e-06, 7.69961725e-03, 1.13099329e-03], + [ 7.38157674e-05, 1.13099329e-03, 7.26783636e-03]]),RTOL*10) + np.testing.assert_equal(reg.name_x, ['CONSTANT', 'RD', 'PS', 'lambda', ' sigma2_v', 'sigma2_1']) + np.testing.assert_equal(reg.name_y, 'HR') def test_regimes(self): regimes = self.db.by_col("SOUTH") - reg = GM_KKP( - self.y, - self.x, - self.w, - full_weights=False, - regimes=regimes, - name_y=self.y_var0, - name_x=self.x_var0, - ) - np.testing.assert_allclose( - reg.betas, - np.array( - [ - [5.25856482], - [3.19249165], - [1.0056967], - [7.94560642], - [3.13931041], - [1.53700634], - [0.35979407], - [22.5650005], - [39.71516708], - ] - ), - RTOL, - ) - np.testing.assert_allclose( - np.sqrt(reg.vm.diagonal()), - np.array([0.158986, 0.157543, 0.104128, 0.165254, 0.117737, 0.136666]), - RTOL, - ) - np.testing.assert_equal( - reg.name_x, - [ - "0_CONSTANT", - "0_RD", - "0_PS", - "1_CONSTANT", - "1_RD", - "1_PS", - "lambda", - " sigma2_v", - "sigma2_1", - ], - ) - np.testing.assert_equal(reg.name_y, "HR") - np.testing.assert_allclose( - reg.chow.regi, - np.array( - [ - [1.420430e02, 9.516507e-33], - [7.311490e-02, 7.868543e-01], - [9.652492e00, 1.890949e-03], - ] - ), - RTOL, - ) - np.testing.assert_allclose(reg.chow.joint[0], 158.7225, RTOL) + reg = GM_KKP(self.y,self.x,self.w,full_weights=False,regimes=regimes, + name_y=self.y_var0, name_x=self.x_var0) + np.testing.assert_allclose(reg.betas,np.array([[ 5.25856482], + [ 3.19249165], + [ 1.0056967 ], + [ 7.94560642], + [ 3.13931041], + [ 1.53700634], + [ 0.35979407], + [22.5650005 ], + [39.71516708]]),RTOL) + np.testing.assert_allclose(np.sqrt(reg.vm.diagonal()),np.array([0.158986, 0.157543, 0.104128, 0.165254, 0.117737, 0.136666]),RTOL) + np.testing.assert_equal(reg.name_x, ['0_CONSTANT', '0_RD', '0_PS', '1_CONSTANT', '1_RD', '1_PS', 'lambda', ' sigma2_v', 'sigma2_1']) + np.testing.assert_equal(reg.name_y, 'HR') + np.testing.assert_allclose(reg.chow.regi,np.array([[1.420430e+02, 9.516507e-33], + [7.311490e-02, 7.868543e-01], + [9.652492e+00, 1.890949e-03]]),RTOL*10) + np.testing.assert_allclose(reg.chow.joint[0],158.7225,RTOL) - -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() + diff --git a/spreg/utils.py b/spreg/utils.py index 19911fb0..ccd74926 100755 --- a/spreg/utils.py +++ b/spreg/utils.py @@ -322,7 +322,7 @@ def _moments2eqs(A1, s, u): return [G, g] -def optim_moments(moments_in, vcX=np.array([0]), all_par=False, start=None): +def optim_moments(moments_in, vcX=np.array([0]), all_par=False, start=None, hard_bound=False): """ Optimization of moments ... @@ -341,7 +341,10 @@ def optim_moments(moments_in, vcX=np.array([0]), all_par=False, start=None): solution or just the 1st. Default is 1st only. start : list List with initial values for the optimization - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. + Returns ------- x, f, d : tuple @@ -386,6 +389,10 @@ def optim_moments(moments_in, vcX=np.array([0]), all_par=False, start=None): bounds = [(-0.99, 0.99), (0.0, None), (0.0, None)] lambdaX = op.fmin_l_bfgs_b(optim_par, start, approx_grad=True, bounds=bounds) + if hard_bound: + if abs(lambdaX[0][0]) >= 0.99: + raise Exception("Spatial parameter was outside the bounds of -0.99 and 0.99") + if all_par: return lambdaX[0] return lambdaX[0][0] From accbf6bf03ca22a155006c30a0b0fd063e35dd75 Mon Sep 17 00:00:00 2001 From: Pedro Amaral Date: Mon, 24 Jun 2024 18:51:21 -0300 Subject: [PATCH 07/10] Adjusting tests tolerances and arguments --- spreg/tests/test_error_sp_hom_sparse.py | 549 +++++++++--------------- spreg/tests/test_twosls_sp_regimes.py | 76 +++- 2 files changed, 277 insertions(+), 348 deletions(-) diff --git a/spreg/tests/test_error_sp_hom_sparse.py b/spreg/tests/test_error_sp_hom_sparse.py index 8c102661..09cb42c8 100644 --- a/spreg/tests/test_error_sp_hom_sparse.py +++ b/spreg/tests/test_error_sp_hom_sparse.py @@ -1,7 +1,7 @@ -""" +''' Unittests for spreg.error_sp_hom module -""" +''' import unittest import libpysal from spreg import error_sp_hom as HOM @@ -9,139 +9,94 @@ import numpy as np import spreg - class BaseGM_Error_Hom_Tester(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) self.X = sparse.csr_matrix(self.X) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HOM.BaseGM_Error_Hom(self.y, self.X, self.w.sparse, A1="hom_sc") - np.testing.assert_array_almost_equal(reg.y[0], np.array([80.467003]), 7) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_array_almost_equal(reg.x[0].toarray()[0], x, 7) - betas = np.array([[47.9478524], [0.70633223], [-0.55595633], [0.41288558]]) - np.testing.assert_array_almost_equal(reg.betas, betas, 7) - np.testing.assert_array_almost_equal(reg.u[0], np.array([27.466734]), 6) - np.testing.assert_array_almost_equal( - reg.e_filtered[0], np.array([32.37298547]), 7 - ) - i_s = "Maximum number of iterations reached." - self.assertAlmostEqual(reg.iter_stop, i_s, 7) - np.testing.assert_array_almost_equal(reg.predy[0], np.array([53.000269]), 6) - self.assertAlmostEqual(reg.n, 49, 7) - self.assertAlmostEqual(reg.k, 3, 7) + reg = HOM.BaseGM_Error_Hom(self.y, self.X, self.w.sparse, A1='hom_sc') + np.testing.assert_array_almost_equal(reg.y[0],np.array([80.467003]),7) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_array_almost_equal(reg.x[0].toarray()[0],x,7) + betas = np.array([[ 47.9478524 ], [ 0.70633223], [ -0.55595633], [ 0.41288558]]) + np.testing.assert_array_almost_equal(reg.betas,betas,7) + np.testing.assert_array_almost_equal(reg.u[0],np.array([27.466734]),6) + np.testing.assert_array_almost_equal(reg.e_filtered[0],np.array([ 32.37298547]),5) + i_s = 'Maximum number of iterations reached.' + self.assertAlmostEqual(reg.iter_stop,i_s,7) + np.testing.assert_array_almost_equal(reg.predy[0],np.array([ 53.000269]),6) + self.assertAlmostEqual(reg.n,49,7) + self.assertAlmostEqual(reg.k,3,7) sig2 = 189.94459439729718 - self.assertAlmostEqual(reg.sig2, sig2) - vm = np.array( - [ - [1.51340717e02, -5.29057506e00, -1.85654540e00, -2.39139054e-03], - [-5.29057506e00, 2.46669610e-01, 5.14259101e-02, 3.19241302e-04], - [-1.85654540e00, 5.14259101e-02, 3.20510550e-02, -5.95640240e-05], - [-2.39139054e-03, 3.19241302e-04, -5.95640240e-05, 3.36690159e-02], - ] - ) - np.testing.assert_array_almost_equal(reg.vm, vm, 6) - xtx = np.array( - [ - [4.90000000e01, 7.04371999e02, 1.72131237e03], - [7.04371999e02, 1.16866734e04, 2.15575320e04], - [1.72131237e03, 2.15575320e04, 7.39058986e04], - ] - ) - np.testing.assert_array_almost_equal(reg.xtx, xtx, 4) - + self.assertAlmostEqual(reg.sig2,sig2,5) + vm = np.array([[ 1.51340717e+02, -5.29057506e+00, -1.85654540e+00, -2.39139054e-03], [ -5.29057506e+00, 2.46669610e-01, 5.14259101e-02, 3.19241302e-04], [ -1.85654540e+00, 5.14259101e-02, 3.20510550e-02, -5.95640240e-05], [ -2.39139054e-03, 3.19241302e-04, -5.95640240e-05, 3.36690159e-02]]) + np.testing.assert_array_almost_equal(reg.vm,vm,6) + xtx = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03], [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04], [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04]]) + np.testing.assert_array_almost_equal(reg.xtx,xtx,4) class GM_Error_Hom_Tester(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T self.X = sparse.csr_matrix(self.X) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HOM.GM_Error_Hom(self.y, self.X, self.w, A1="hom_sc") - np.testing.assert_array_almost_equal(reg.y[0], np.array([80.467003]), 7) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_array_almost_equal(reg.x[0].toarray()[0], x, 7) - betas = np.array([[47.9478524], [0.70633223], [-0.55595633], [0.41288558]]) - np.testing.assert_array_almost_equal(reg.betas, betas, 7) - np.testing.assert_array_almost_equal(reg.u[0], np.array([27.46673388]), 6) - np.testing.assert_array_almost_equal( - reg.e_filtered[0], np.array([32.37298547]), 7 - ) - np.testing.assert_array_almost_equal(reg.predy[0], np.array([53.00026912]), 6) - self.assertAlmostEqual(reg.n, 49, 7) - self.assertAlmostEqual(reg.k, 3, 7) - vm = np.array( - [ - [1.51340717e02, -5.29057506e00, -1.85654540e00, -2.39139054e-03], - [-5.29057506e00, 2.46669610e-01, 5.14259101e-02, 3.19241302e-04], - [-1.85654540e00, 5.14259101e-02, 3.20510550e-02, -5.95640240e-05], - [-2.39139054e-03, 3.19241302e-04, -5.95640240e-05, 3.36690159e-02], - ] - ) - np.testing.assert_array_almost_equal(reg.vm, vm, 6) - i_s = "Maximum number of iterations reached." - self.assertAlmostEqual(reg.iter_stop, i_s, 7) - self.assertAlmostEqual(reg.iteration, 1, 7) + reg = HOM.GM_Error_Hom(self.y, self.X, self.w, A1='hom_sc') + np.testing.assert_array_almost_equal(reg.y[0],np.array([80.467003]),7) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_array_almost_equal(reg.x[0].toarray()[0],x,7) + betas = np.array([[ 47.9478524 ], [ 0.70633223], [ -0.55595633], [ 0.41288558]]) + np.testing.assert_array_almost_equal(reg.betas,betas,7) + np.testing.assert_array_almost_equal(reg.u[0],np.array([27.46673388]),6) + np.testing.assert_array_almost_equal(reg.e_filtered[0],np.array([ 32.37298547]),5) + np.testing.assert_array_almost_equal(reg.predy[0],np.array([ 53.00026912]),6) + self.assertAlmostEqual(reg.n,49,7) + self.assertAlmostEqual(reg.k,3,7) + vm = np.array([[ 1.51340717e+02, -5.29057506e+00, -1.85654540e+00, -2.39139054e-03], [ -5.29057506e+00, 2.46669610e-01, 5.14259101e-02, 3.19241302e-04], [ -1.85654540e+00, 5.14259101e-02, 3.20510550e-02, -5.95640240e-05], [ -2.39139054e-03, 3.19241302e-04, -5.95640240e-05, 3.36690159e-02]]) + np.testing.assert_array_almost_equal(reg.vm,vm,6) + i_s = 'Maximum number of iterations reached.' + self.assertAlmostEqual(reg.iter_stop,i_s,7) + self.assertAlmostEqual(reg.iteration,1,7) my = 38.436224469387746 - self.assertAlmostEqual(reg.mean_y, my) + self.assertAlmostEqual(reg.mean_y,my) std_y = 18.466069465206047 - self.assertAlmostEqual(reg.std_y, std_y) + self.assertAlmostEqual(reg.std_y,std_y) pr2 = 0.34950977055969729 - self.assertAlmostEqual(reg.pr2, pr2) + self.assertAlmostEqual(reg.pr2,pr2) sig2 = 189.94459439729718 - self.assertAlmostEqual(reg.sig2, sig2) - std_err = np.array([12.30206149, 0.49665844, 0.17902808, 0.18349119]) - np.testing.assert_array_almost_equal(reg.std_err, std_err, 6) - z_stat = np.array( - [ - [3.89754616e00, 9.71723059e-05], - [1.42216900e00, 1.54977196e-01], - [-3.10541409e00, 1.90012806e-03], - [2.25016500e00, 2.44384731e-02], - ] - ) - np.testing.assert_array_almost_equal(reg.z_stat, z_stat, 6) - xtx = np.array( - [ - [4.90000000e01, 7.04371999e02, 1.72131237e03], - [7.04371999e02, 1.16866734e04, 2.15575320e04], - [1.72131237e03, 2.15575320e04, 7.39058986e04], - ] - ) - np.testing.assert_array_almost_equal(reg.xtx, xtx, 4) + self.assertAlmostEqual(reg.sig2,sig2,5) + std_err = np.array([ 12.30206149, 0.49665844, 0.17902808, 0.18349119]) + np.testing.assert_array_almost_equal(reg.std_err,std_err,6) + z_stat = np.array([[ 3.89754616e+00, 9.71723059e-05], [ 1.42216900e+00, 1.54977196e-01], [ -3.10541409e+00, 1.90012806e-03], [ 2.25016500e+00, 2.44384731e-02]]) + np.testing.assert_array_almost_equal(reg.z_stat,z_stat,6) + xtx = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03], [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04], [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04]]) + np.testing.assert_array_almost_equal(reg.xtx,xtx,4) class BaseGM_Endog_Error_Hom_Tester(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) self.X = sparse.csr_matrix(self.X) yd = [] yd.append(db.by_col("CRIME")) @@ -149,73 +104,52 @@ def setUp(self): q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HOM.BaseGM_Endog_Error_Hom( - self.y, self.X, self.yd, self.q, self.w.sparse, A1="hom_sc" - ) - np.testing.assert_array_almost_equal(reg.y[0], np.array([80.467003]), 7) - x = np.array([1.0, 19.531]) - np.testing.assert_array_almost_equal(reg.x[0].toarray()[0], x, 7) - z = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_array_almost_equal(reg.z[0].toarray()[0], z, 7) - h = np.array([1.0, 19.531, 5.03]) - np.testing.assert_array_almost_equal(reg.h[0].toarray()[0], h, 7) - yend = np.array([15.72598]) - np.testing.assert_array_almost_equal(reg.yend[0], yend, 7) - q = np.array([5.03]) - np.testing.assert_array_almost_equal(reg.q[0], q, 7) - betas = np.array([[55.36575166], [0.46432416], [-0.66904404], [0.43205526]]) - np.testing.assert_array_almost_equal(reg.betas, betas, 6) - u = np.array([26.55390939]) - np.testing.assert_array_almost_equal(reg.u[0], u, 6) - np.testing.assert_array_almost_equal( - reg.e_filtered[0], np.array([31.74114306]), 7 - ) - predy = np.array([53.91309361]) - np.testing.assert_array_almost_equal(reg.predy[0], predy, 6) - self.assertAlmostEqual(reg.n, 49, 7) - self.assertAlmostEqual(reg.k, 3, 7) + reg = HOM.BaseGM_Endog_Error_Hom(self.y, self.X, self.yd, self.q, self.w.sparse, A1='hom_sc') + np.testing.assert_array_almost_equal(reg.y[0],np.array([ 80.467003]),7) + x = np.array([ 1. , 19.531]) + np.testing.assert_array_almost_equal(reg.x[0].toarray()[0],x,7) + z = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_array_almost_equal(reg.z[0].toarray()[0],z,7) + h = np.array([ 1. , 19.531, 5.03 ]) + np.testing.assert_array_almost_equal(reg.h[0].toarray()[0],h,7) + yend = np.array([ 15.72598]) + np.testing.assert_array_almost_equal(reg.yend[0],yend,7) + q = np.array([ 5.03]) + np.testing.assert_array_almost_equal(reg.q[0],q,7) + betas = np.array([[ 55.36575166], [ 0.46432416], [ -0.66904404], [ 0.43205526]]) + np.testing.assert_array_almost_equal(reg.betas,betas,6) + u = np.array([ 26.55390939]) + np.testing.assert_array_almost_equal(reg.u[0],u,6) + np.testing.assert_array_almost_equal(reg.e_filtered[0],np.array([ 31.74114306]),5) + predy = np.array([ 53.91309361]) + np.testing.assert_array_almost_equal(reg.predy[0],predy,6) + self.assertAlmostEqual(reg.n,49,7) + self.assertAlmostEqual(reg.k,3,7) sig2 = 190.59435238060928 - self.assertAlmostEqual(reg.sig2, sig2) - vm = np.array( - [ - [5.52064057e02, -1.61264555e01, -8.86360735e00, 1.04251912e00], - [-1.61264555e01, 5.44898242e-01, 2.39518645e-01, -1.88092950e-02], - [-8.86360735e00, 2.39518645e-01, 1.55501840e-01, -2.18638648e-02], - [1.04251912e00, -1.88092950e-02, -2.18638648e-02, 3.71222222e-02], - ] - ) - np.testing.assert_array_almost_equal(reg.vm, vm, 6) - i_s = "Maximum number of iterations reached." - self.assertAlmostEqual(reg.iter_stop, i_s, 7) + self.assertAlmostEqual(reg.sig2,sig2,4) + vm = np.array([[ 5.52064057e+02, -1.61264555e+01, -8.86360735e+00, 1.04251912e+00], [ -1.61264555e+01, 5.44898242e-01, 2.39518645e-01, -1.88092950e-02], [ -8.86360735e+00, 2.39518645e-01, 1.55501840e-01, -2.18638648e-02], [ 1.04251912e+00, -1.88092950e-02, -2.18638648e-02, 3.71222222e-02]]) + np.testing.assert_array_almost_equal(reg.vm,vm,4) + i_s = 'Maximum number of iterations reached.' + self.assertAlmostEqual(reg.iter_stop,i_s,7) its = 1 - self.assertAlmostEqual(reg.iteration, its, 7) + self.assertAlmostEqual(reg.iteration,its,7) my = 38.436224469387746 - self.assertAlmostEqual(reg.mean_y, my) + self.assertAlmostEqual(reg.mean_y,my) std_y = 18.466069465206047 - self.assertAlmostEqual(reg.std_y, std_y) + self.assertAlmostEqual(reg.std_y,std_y) sig2 = 0 - # self.assertAlmostEqual(reg.sig2,sig2) - hth = np.array( - [ - [49.0, 704.371999, 139.75], - [704.371999, 11686.67338121, 2246.12800625], - [139.75, 2246.12800625, 498.5851], - ] - ) - np.testing.assert_array_almost_equal(reg.hth, hth, 4) - + #self.assertAlmostEqual(reg.sig2,sig2) + hth = np.array([[ 49. , 704.371999 , 139.75 ], [ 704.371999 , 11686.67338121, 2246.12800625], [ 139.75 , 2246.12800625, 498.5851]]) + np.testing.assert_array_almost_equal(reg.hth,hth,4) class GM_Endog_Error_Hom_Tester(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T @@ -226,227 +160,162 @@ def setUp(self): q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HOM.GM_Endog_Error_Hom( - self.y, self.X, self.yd, self.q, self.w, A1="hom_sc" - ) - np.testing.assert_array_almost_equal(reg.y[0], np.array([80.467003]), 7) - x = np.array([1.0, 19.531]) - np.testing.assert_array_almost_equal(reg.x[0].toarray()[0], x, 7) - z = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_array_almost_equal(reg.z[0].toarray()[0], z, 7) - h = np.array([1.0, 19.531, 5.03]) - np.testing.assert_array_almost_equal(reg.h[0].toarray()[0], h, 7) - yend = np.array([15.72598]) - np.testing.assert_array_almost_equal(reg.yend[0], yend, 7) - q = np.array([5.03]) - np.testing.assert_array_almost_equal(reg.q[0], q, 7) - betas = np.array([[55.36575166], [0.46432416], [-0.66904404], [0.43205526]]) - np.testing.assert_array_almost_equal(reg.betas, betas, 6) - u = np.array([26.55390939]) - np.testing.assert_array_almost_equal(reg.u[0], u, 6) - np.testing.assert_array_almost_equal( - reg.e_filtered[0], np.array([31.74114306]), 7 - ) - predy = np.array([53.91309361]) - np.testing.assert_array_almost_equal(reg.predy[0], predy, 6) - self.assertAlmostEqual(reg.n, 49, 7) - self.assertAlmostEqual(reg.k, 3, 7) - vm = np.array( - [ - [5.52064057e02, -1.61264555e01, -8.86360735e00, 1.04251912e00], - [-1.61264555e01, 5.44898242e-01, 2.39518645e-01, -1.88092950e-02], - [-8.86360735e00, 2.39518645e-01, 1.55501840e-01, -2.18638648e-02], - [1.04251912e00, -1.88092950e-02, -2.18638648e-02, 3.71222222e-02], - ] - ) - np.testing.assert_array_almost_equal(reg.vm, vm, 6) - i_s = "Maximum number of iterations reached." - self.assertAlmostEqual(reg.iter_stop, i_s, 7) + reg = HOM.GM_Endog_Error_Hom(self.y, self.X, self.yd, self.q, self.w, A1='hom_sc') + np.testing.assert_array_almost_equal(reg.y[0],np.array([ 80.467003]),7) + x = np.array([ 1. , 19.531]) + np.testing.assert_array_almost_equal(reg.x[0].toarray()[0],x,7) + z = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_array_almost_equal(reg.z[0].toarray()[0],z,7) + h = np.array([ 1. , 19.531, 5.03 ]) + np.testing.assert_array_almost_equal(reg.h[0].toarray()[0],h,7) + yend = np.array([ 15.72598]) + np.testing.assert_array_almost_equal(reg.yend[0],yend,7) + q = np.array([ 5.03]) + np.testing.assert_array_almost_equal(reg.q[0],q,7) + betas = np.array([[ 55.36575166], [ 0.46432416], [ -0.66904404], [ 0.43205526]]) + np.testing.assert_array_almost_equal(reg.betas,betas,6) + u = np.array([ 26.55390939]) + np.testing.assert_array_almost_equal(reg.u[0],u,6) + np.testing.assert_array_almost_equal(reg.e_filtered[0],np.array([ 31.74114]),5) + predy = np.array([ 53.91309361]) + np.testing.assert_array_almost_equal(reg.predy[0],predy,6) + self.assertAlmostEqual(reg.n,49,7) + self.assertAlmostEqual(reg.k,3,7) + vm = np.array([[ 5.52064057e+02, -1.61264555e+01, -8.86360735e+00, 1.04251912e+00], [ -1.61264555e+01, 5.44898242e-01, 2.39518645e-01, -1.88092950e-02], [ -8.86360735e+00, 2.39518645e-01, 1.55501840e-01, -2.18638648e-02], [ 1.04251912e+00, -1.88092950e-02, -2.18638648e-02, 3.71222222e-02]]) + np.testing.assert_array_almost_equal(reg.vm,vm,4) + i_s = 'Maximum number of iterations reached.' + self.assertAlmostEqual(reg.iter_stop,i_s,7) its = 1 - self.assertAlmostEqual(reg.iteration, its, 7) + self.assertAlmostEqual(reg.iteration,its,7) my = 38.436224469387746 - self.assertAlmostEqual(reg.mean_y, my) + self.assertAlmostEqual(reg.mean_y,my) std_y = 18.466069465206047 - self.assertAlmostEqual(reg.std_y, std_y) + self.assertAlmostEqual(reg.std_y,std_y) pr2 = 0.34647366525657419 - self.assertAlmostEqual(reg.pr2, pr2) + self.assertAlmostEqual(reg.pr2,pr2) sig2 = 190.59435238060928 - self.assertAlmostEqual(reg.sig2, sig2) - # std_err - std_err = np.array([23.49604343, 0.73817223, 0.39433722, 0.19267128]) - np.testing.assert_array_almost_equal(reg.std_err, std_err, 6) - z_stat = np.array( - [ - [2.35638617, 0.01845372], - [0.62901874, 0.52933679], - [-1.69662923, 0.08976678], - [2.24244556, 0.02493259], - ] - ) - np.testing.assert_array_almost_equal(reg.z_stat, z_stat, 6) + self.assertAlmostEqual(reg.sig2,sig2,4) + #std_err + std_err = np.array([ 23.49604343, 0.73817223, 0.39433722, 0.19267128]) + np.testing.assert_array_almost_equal(reg.std_err,std_err,4) + z_stat = np.array([[ 2.35638617, 0.01845372], [ 0.62901874, 0.52933679], [-1.69662923, 0.08976678], [ 2.24244556, 0.02493259]]) + np.testing.assert_array_almost_equal(reg.z_stat,z_stat,6) class BaseGM_Combo_Hom_Tester(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): yd2, q2 = spreg.utils.set_endog(self.y, self.X, self.w, None, None, 1, True) - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) self.X = sparse.csr_matrix(self.X) - reg = HOM.BaseGM_Combo_Hom( - self.y, self.X, yend=yd2, q=q2, w=self.w.sparse, A1="hom_sc" - ) - np.testing.assert_array_almost_equal(reg.y[0], np.array([80.467003]), 7) - x = np.array([1.0, 19.531]) - np.testing.assert_array_almost_equal(reg.x[0].toarray()[0], x, 7) - betas = np.array([[10.12541428], [1.56832263], [0.15132076], [0.21033397]]) - np.testing.assert_array_almost_equal(reg.betas, betas, 7) - np.testing.assert_array_almost_equal(reg.u[0], np.array([34.3450723]), 7) - np.testing.assert_array_almost_equal( - reg.e_filtered[0], np.array([36.6149682]), 7 - ) - np.testing.assert_array_almost_equal(reg.predy[0], np.array([46.1219307]), 7) - self.assertAlmostEqual(reg.n, 49, 7) - self.assertAlmostEqual(reg.k, 3, 7) - vm = np.array( - [ - [2.33694742e02, -6.66856869e-01, -5.58304254e00, 4.85488380e00], - [-6.66856869e-01, 1.94241504e-01, -5.42327138e-02, 5.37225570e-02], - [-5.58304254e00, -5.42327138e-02, 1.63860721e-01, -1.44425498e-01], - [4.85488380e00, 5.37225570e-02, -1.44425498e-01, 1.78622255e-01], - ] - ) - np.testing.assert_array_almost_equal(reg.vm, vm, 6) - z = np.array([1.0, 19.531, 35.4585005]) - np.testing.assert_array_almost_equal(reg.z[0].toarray()[0], z, 7) - h = np.array([1.0, 19.531, 18.594]) - np.testing.assert_array_almost_equal(reg.h[0].toarray()[0], h, 7) - yend = np.array([35.4585005]) - np.testing.assert_array_almost_equal(reg.yend[0], yend, 7) - q = np.array([18.594]) - np.testing.assert_array_almost_equal(reg.q[0], q, 7) - i_s = "Maximum number of iterations reached." - self.assertAlmostEqual(reg.iter_stop, i_s, 7) + reg = HOM.BaseGM_Combo_Hom(self.y, self.X, yend=yd2, q=q2, w=self.w.sparse, A1='hom_sc') + np.testing.assert_array_almost_equal(reg.y[0],np.array([80.467003]),7) + x = np.array([ 1. , 19.531]) + np.testing.assert_array_almost_equal(reg.x[0].toarray()[0],x,7) + betas = np.array([[ 10.12541428], [ 1.56832263], [ 0.15132076], [ 0.21033397]]) + np.testing.assert_array_almost_equal(reg.betas,betas,7) + np.testing.assert_array_almost_equal(reg.u[0],np.array([34.3450723]),7) + np.testing.assert_array_almost_equal(reg.e_filtered[0],np.array([ 36.6149682]),7) + np.testing.assert_array_almost_equal(reg.predy[0],np.array([ 46.1219307]),7) + self.assertAlmostEqual(reg.n,49,7) + self.assertAlmostEqual(reg.k,3,7) + vm = np.array([[ 2.33694742e+02, -6.66856869e-01, -5.58304254e+00, 4.85488380e+00], [ -6.66856869e-01, 1.94241504e-01, -5.42327138e-02, 5.37225570e-02], [ -5.58304254e+00, -5.42327138e-02, 1.63860721e-01, -1.44425498e-01], [ 4.85488380e+00, 5.37225570e-02, -1.44425498e-01, 1.78622255e-01]]) + np.testing.assert_array_almost_equal(reg.vm,vm,6) + z = np.array([ 1. , 19.531 , 35.4585005]) + np.testing.assert_array_almost_equal(reg.z[0].toarray()[0],z,7) + h = np.array([ 1. , 19.531, 18.594]) + np.testing.assert_array_almost_equal(reg.h[0].toarray()[0],h,7) + yend = np.array([ 35.4585005]) + np.testing.assert_array_almost_equal(reg.yend[0],yend,7) + q = np.array([ 18.594]) + np.testing.assert_array_almost_equal(reg.q[0],q,7) + i_s = 'Maximum number of iterations reached.' + self.assertAlmostEqual(reg.iter_stop,i_s,7) its = 1 - self.assertAlmostEqual(reg.iteration, its, 7) + self.assertAlmostEqual(reg.iteration,its,7) my = 38.436224469387746 - self.assertAlmostEqual(reg.mean_y, my) + self.assertAlmostEqual(reg.mean_y,my) std_y = 18.466069465206047 - self.assertAlmostEqual(reg.std_y, std_y) + self.assertAlmostEqual(reg.std_y,std_y) sig2 = 232.22680644168395 - self.assertAlmostEqual(reg.sig2, sig2, places=6) - hth = np.array( - [ - [49.0, 704.371999, 724.7435916], - [704.371999, 11686.67338121, 11092.519988], - [724.7435916, 11092.519988, 11614.62257048], - ] - ) - np.testing.assert_array_almost_equal(reg.hth, hth, 4) + self.assertAlmostEqual(reg.sig2,sig2, places=6) + hth = np.array([[ 49. , 704.371999 , 724.7435916 ], [ 704.371999 , 11686.67338121, 11092.519988 ], [ 724.7435916 , 11092.519988 , 11614.62257048]]) + np.testing.assert_array_almost_equal(reg.hth,hth,4) class GM_Combo_Hom_Tester(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T self.X = sparse.csr_matrix(self.X) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HOM.GM_Combo_Hom(self.y, self.X, w=self.w, A1="hom_sc") - np.testing.assert_array_almost_equal(reg.y[0], np.array([80.467003]), 7) - x = np.array([1.0, 19.531]) - np.testing.assert_array_almost_equal(reg.x[0].toarray()[0], x, 7) - betas = np.array([[10.12541428], [1.56832263], [0.15132076], [0.21033397]]) - np.testing.assert_array_almost_equal(reg.betas, betas, 7) - np.testing.assert_array_almost_equal(reg.u[0], np.array([34.3450723]), 7) - np.testing.assert_array_almost_equal( - reg.e_filtered[0], np.array([36.6149682]), 7 - ) - np.testing.assert_array_almost_equal(reg.e_pred[0], np.array([32.90372983]), 7) - np.testing.assert_array_almost_equal(reg.predy[0], np.array([46.1219307]), 7) - np.testing.assert_array_almost_equal(reg.predy_e[0], np.array([47.56327317]), 7) - self.assertAlmostEqual(reg.n, 49, 7) - self.assertAlmostEqual(reg.k, 3, 7) - z = np.array([1.0, 19.531, 35.4585005]) - np.testing.assert_array_almost_equal(reg.z[0].toarray()[0], z, 7) - h = np.array([1.0, 19.531, 18.594]) - np.testing.assert_array_almost_equal(reg.h[0].toarray()[0], h, 7) - yend = np.array([35.4585005]) - np.testing.assert_array_almost_equal(reg.yend[0], yend, 7) - q = np.array([18.594]) - np.testing.assert_array_almost_equal(reg.q[0].toarray()[0], q, 7) - i_s = "Maximum number of iterations reached." - self.assertAlmostEqual(reg.iter_stop, i_s, 7) - self.assertAlmostEqual(reg.iteration, 1, 7) + reg = HOM.GM_Combo_Hom(self.y, self.X, w=self.w, A1='hom_sc') + np.testing.assert_array_almost_equal(reg.y[0],np.array([80.467003]),7) + x = np.array([ 1. , 19.531]) + np.testing.assert_array_almost_equal(reg.x[0].toarray()[0],x,7) + betas = np.array([[ 10.12541428], [ 1.56832263], [ 0.15132076], [ 0.21033397]]) + np.testing.assert_array_almost_equal(reg.betas,betas,7) + np.testing.assert_array_almost_equal(reg.u[0],np.array([34.3450723]),7) + np.testing.assert_array_almost_equal(reg.e_filtered[0],np.array([ 36.6149682]),7) + np.testing.assert_array_almost_equal(reg.e_pred[0],np.array([ 32.90372983]),7) + np.testing.assert_array_almost_equal(reg.predy[0],np.array([ 46.1219307]),7) + np.testing.assert_array_almost_equal(reg.predy_e[0],np.array([47.56327317]),7) + self.assertAlmostEqual(reg.n,49,7) + self.assertAlmostEqual(reg.k,3,7) + z = np.array([ 1. , 19.531 , 35.4585005]) + np.testing.assert_array_almost_equal(reg.z[0].toarray()[0],z,7) + h = np.array([ 1. , 19.531, 18.594]) + np.testing.assert_array_almost_equal(reg.h[0].toarray()[0],h,7) + yend = np.array([ 35.4585005]) + np.testing.assert_array_almost_equal(reg.yend[0],yend,7) + q = np.array([ 18.594]) + np.testing.assert_array_almost_equal(reg.q[0].toarray()[0],q,7) + i_s = 'Maximum number of iterations reached.' + self.assertAlmostEqual(reg.iter_stop,i_s,7) + self.assertAlmostEqual(reg.iteration,1,7) my = 38.436224469387746 - self.assertAlmostEqual(reg.mean_y, my) + self.assertAlmostEqual(reg.mean_y,my) std_y = 18.466069465206047 - self.assertAlmostEqual(reg.std_y, std_y) + self.assertAlmostEqual(reg.std_y,std_y) pr2 = 0.28379825632694394 - self.assertAlmostEqual(reg.pr2, pr2) + self.assertAlmostEqual(reg.pr2,pr2) pr2_e = 0.25082892555141506 - self.assertAlmostEqual(reg.pr2_e, pr2_e) + self.assertAlmostEqual(reg.pr2_e,pr2_e) sig2 = 232.22680644168395 - self.assertAlmostEqual(reg.sig2, sig2, places=6) - std_err = np.array([15.28707761, 0.44072838, 0.40479714, 0.42263726]) - np.testing.assert_array_almost_equal(reg.std_err, std_err, 6) - z_stat = np.array( - [ - [6.62351206e-01, 5.07746167e-01], - [3.55847888e00, 3.73008780e-04], - [3.73818749e-01, 7.08539170e-01], - [4.97670189e-01, 6.18716523e-01], - ] - ) - np.testing.assert_array_almost_equal(reg.z_stat, z_stat, 6) - vm = np.array( - [ - [2.33694742e02, -6.66856869e-01, -5.58304254e00, 4.85488380e00], - [-6.66856869e-01, 1.94241504e-01, -5.42327138e-02, 5.37225570e-02], - [-5.58304254e00, -5.42327138e-02, 1.63860721e-01, -1.44425498e-01], - [4.85488380e00, 5.37225570e-02, -1.44425498e-01, 1.78622255e-01], - ] - ) - np.testing.assert_array_almost_equal(reg.vm, vm, 6) - + self.assertAlmostEqual(reg.sig2,sig2, places=6) + std_err = np.array([ 15.28707761, 0.44072838, 0.40479714, 0.42263726]) + np.testing.assert_array_almost_equal(reg.std_err,std_err,6) + z_stat = np.array([[ 6.62351206e-01, 5.07746167e-01], [ 3.55847888e+00, 3.73008780e-04], [ 3.73818749e-01, 7.08539170e-01], [ 4.97670189e-01, 6.18716523e-01]]) + np.testing.assert_array_almost_equal(reg.z_stat,z_stat,6) + vm = np.array([[ 2.33694742e+02, -6.66856869e-01, -5.58304254e+00, 4.85488380e+00], [ -6.66856869e-01, 1.94241504e-01, -5.42327138e-02, 5.37225570e-02], [ -5.58304254e+00, -5.42327138e-02, 1.63860721e-01, -1.44425498e-01], [ 4.85488380e+00, 5.37225570e-02, -1.44425498e-01, 1.78622255e-01]]) + np.testing.assert_array_almost_equal(reg.vm,vm,6) suite = unittest.TestSuite() -test_classes = [ - BaseGM_Error_Hom_Tester, - GM_Error_Hom_Tester, - BaseGM_Endog_Error_Hom_Tester, - GM_Endog_Error_Hom_Tester, - BaseGM_Combo_Hom_Tester, - GM_Combo_Hom_Tester, -] +test_classes = [BaseGM_Error_Hom_Tester, GM_Error_Hom_Tester,\ + BaseGM_Endog_Error_Hom_Tester, GM_Endog_Error_Hom_Tester, \ + BaseGM_Combo_Hom_Tester, GM_Combo_Hom_Tester] for i in test_classes: a = unittest.TestLoader().loadTestsFromTestCase(i) suite.addTest(a) -if __name__ == "__main__": +if __name__ == '__main__': runner = unittest.TextTestRunner() runner.run(suite) + diff --git a/spreg/tests/test_twosls_sp_regimes.py b/spreg/tests/test_twosls_sp_regimes.py index f763ac74..3d299381 100644 --- a/spreg/tests/test_twosls_sp_regimes.py +++ b/spreg/tests/test_twosls_sp_regimes.py @@ -22,7 +22,7 @@ def test___init__(self): X.append(self.db.by_col("INC")) X.append(self.db.by_col("HOVAL")) self.X = np.array(X).T - reg = GM_Lag_Regimes(self.y, self.X, self.regimes, w=self.w, sig2n_k=True, regime_lag_sep=False, regime_err_sep=False) + reg = GM_Lag_Regimes(self.y, self.X, self.regimes, w=self.w, sig2n_k=True, regime_lag_sep=False, regime_err_sep=False, robust=None) betas = np.array([[ 45.14892906], [ -1.42593383], [ -0.11501037], @@ -121,7 +121,7 @@ def test_init_discbd(self): yd = np.reshape(yd, (49,1)) q = np.array(self.db.by_col("DISCBD")) q = np.reshape(q, (49,1)) - reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, lag_q=False, w=self.w, sig2n_k=True, regime_lag_sep=False, regime_err_sep=False) + reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, lag_q=False, w=self.w, sig2n_k=True, regime_lag_sep=False, regime_err_sep=False, robust=None) tbetas = np.array([[ 42.7266306 ], [ -0.15552345], [ 37.70545276], @@ -162,7 +162,7 @@ def test_lag_q(self): yd = np.reshape(yd, (49,1)) q = np.array(self.db.by_col("DISCBD")) q = np.reshape(q, (49,1)) - reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w, sig2n_k=True, regime_lag_sep=False, regime_err_sep=False) + reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w, sig2n_k=True, regime_lag_sep=False, regime_err_sep=False, robust=None) tbetas = np.array([[ 37.87698329], [ -0.89426982], [ 31.4714777 ], @@ -187,7 +187,7 @@ def test_all_regi(self): yd = np.reshape(yd, (49,1)) q = np.array(self.db.by_col("DISCBD")) q = np.reshape(q, (49,1)) - reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w, regime_lag_sep=False, regime_err_sep=False) + reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w, regime_lag_sep=False, regime_err_sep=False, robust=None) tbetas = np.array([[ 37.87698329, -0.89426982, 31.4714777 , -0.71640525, -0.28494432, -0.2294271 , 0.62996544]]) np.testing.assert_allclose(tbetas, reg.betas.T,RTOL) @@ -228,7 +228,7 @@ def test_all_regi_sig2(self): w = libpysal.weights.util.lat2W(latt,latt) w.transform='r' regi = [0]*(n//2) + [1]*(n//2) - model = GM_Lag_Regimes(y, x1, regi, q=q, yend=x2, w=w, regime_lag_sep=True, regime_err_sep=True) + model = GM_Lag_Regimes(y, x1, regi, q=q, yend=x2, w=w, regime_lag_sep=True, regime_err_sep=True, robust=None) w1 = libpysal.weights.util.lat2W(latt//2,latt) w1.transform='r' model1 = GM_Lag(y[0:(n//2)].reshape((n//2),1), x1[0:(n//2)],yend=x2[0:(n//2)], q=q[0:(n//2)], w=w1) @@ -244,7 +244,7 @@ def test_all_regi_sig2(self): yd = np.reshape(yd, (49,1)) q = np.array(self.db.by_col("DISCBD")) q = np.reshape(q, (49,1)) - reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w,regime_lag_sep=True, regime_err_sep = True) + reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w,regime_lag_sep=True, regime_err_sep = True, robust=None) tbetas = np.array([[ 42.35827477], [ -0.09472413], [ -0.68794223], @@ -287,7 +287,7 @@ def test_fixed_const(self): yd = np.reshape(yd, (49,1)) q = np.array(self.db.by_col("DISCBD")) q = np.reshape(q, (49,1)) - reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w, constant_regi='one', regime_lag_sep=False, regime_err_sep=False) + reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w, constant_regi='one', regime_lag_sep=False, regime_err_sep=False, robust=None) tbetas = np.array([[ -0.37658823], [ -0.9666079 ], [ 35.5445944 ], @@ -328,7 +328,9 @@ def test_names(self): q_var = ['DISCBD'] q = np.array([self.db.by_col(name) for name in q_var]).T r_var = 'NSA' - reg = GM_Lag_Regimes(self.y, x, self.regimes, yend=yd, q=q, w=self.w, name_y=y_var, name_x=x_var, name_yend=yd_var, name_q=q_var, name_regimes=r_var, name_ds='columbus', name_w='columbus.gal', regime_lag_sep=False, regime_err_sep=False) + reg = GM_Lag_Regimes(self.y, x, self.regimes, yend=yd, q=q, w=self.w, name_y=y_var, name_x=x_var, + name_yend=yd_var, name_q=q_var, name_regimes=r_var, name_ds='columbus', + name_w='columbus.gal', regime_lag_sep=False, regime_err_sep=False, robust=None) betas = np.array([[ 37.87698329], [ -0.89426982], [ 31.4714777 ], @@ -350,5 +352,63 @@ def test_names(self): self.assertListEqual(reg.name_q, ['0_DISCBD', '0_W_INC', '0_W_DISCBD', '1_DISCBD', '1_W_INC', '1_W_DISCBD']) self.assertEqual(reg.name_y, y_var) + def test_slx(self): + X = [] + X.append(self.db.by_col("INC")) + X.append(self.db.by_col("HOVAL")) + self.X = np.array(X).T + reg = GM_Lag_Regimes(self.y, self.X, self.regimes, w=self.w, slx_lags=1, regime_lag_sep=False, + regime_err_sep=False, spat_diag=True) + betas = np.array([[ 8.18931398e+01], + [-1.53395435e+00], + [-1.69757478e-01], + [-2.27789212e+00], + [ 5.39079719e-01], + [ 7.51699174e+01], + [-1.08739064e+00], + [-2.83520279e-01], + [-1.68207712e+00], + [ 2.74687447e-01], + [-3.46101374e-02]]) + np.testing.assert_allclose(reg.betas, betas,RTOL) + vm = np.array( [[ 1.73599745e+03, -9.34710692e+00, -1.24643977e+00, -6.47037384e+01, + 7.12544407e+00, 1.67217894e+03, -3.05066669e+00, -2.35566285e+00, + -3.66089456e+01, -4.16736383e+00, -2.36649704e+01], + [-9.34710692e+00, 2.76627235e-01, -4.82833143e-02, 8.93797856e-02, + 3.01134955e-02, -9.78627573e+00, 2.41397355e-02, 1.18016819e-02, + 2.19536352e-01, 2.40036703e-02, 1.36868186e-01], + [-1.24643977e+00, -4.82833143e-02, 2.49271317e-02, 1.08816895e-01, + -3.05204771e-02, -8.06738318e-01, 6.26670530e-04, 1.40330458e-03, + 1.69512409e-02, 2.06235513e-03, 1.16361002e-02], + [-6.47037384e+01, 8.93797856e-02, 1.08816895e-01, 3.37057241e+00, + -6.18148061e-01, -6.02788288e+01, 1.07776025e-01, 8.56099990e-02, + 1.31783651e+00, 1.50360008e-01, 8.53645284e-01], + [ 7.12544407e+00, 3.01134955e-02, -3.05204771e-02, -6.18148061e-01, + 1.79091963e-01, 6.63593281e+00, -1.64138641e-02, -7.98832541e-03, + -1.48902330e-01, -1.62737808e-02, -9.27966724e-02], + [ 1.67217894e+03, -9.78627573e+00, -8.06738318e-01, -6.02788288e+01, + 6.63593281e+00, 1.77071537e+03, -7.45841412e+00, -1.64359023e+00, + -3.92564157e+01, -4.79661230e+00, -2.37950208e+01], + [-3.05066669e+00, 2.41397355e-02, 6.26670530e-04, 1.07776025e-01, + -1.64138641e-02, -7.45841412e+00, 4.80396971e-01, -9.88376849e-02, + 8.60302399e-02, 4.92161147e-02, 4.74983701e-02], + [-2.35566285e+00, 1.18016819e-02, 1.40330458e-03, 8.56099990e-02, + -7.98832541e-03, -1.64359023e+00, -9.88376849e-02, 4.80027099e-02, + -2.23775047e-02, 6.77837002e-03, 3.22304374e-02], + [-3.66089456e+01, 2.19536352e-01, 1.69512409e-02, 1.31783651e+00, + -1.48902330e-01, -3.92564157e+01, 8.60302399e-02, -2.23775047e-02, + 1.50194231e+00, -1.84011183e-02, 5.24380589e-01], + [-4.16736383e+00, 2.40036703e-02, 2.06235513e-03, 1.50360008e-01, + -1.62737808e-02, -4.79661230e+00, 4.92161147e-02, 6.77837002e-03, + -1.84011183e-02, 4.72613924e-02, 5.90507313e-02], + [-2.36649704e+01, 1.36868186e-01, 1.16361002e-02, 8.53645284e-01, + -9.27966724e-02, -2.37950208e+01, 4.74983701e-02, 3.22304374e-02, + 5.24380589e-01, 5.90507313e-02, 3.35692100e-01]]) + np.testing.assert_allclose(reg.vm, vm,RTOL) + ak_test = np.array([0.528338, 0.467306]) + np.testing.assert_allclose(reg.ak_test, ak_test,RTOL) + cfh_test = np.array([8.094023, 0.088194]) + np.testing.assert_allclose(reg.cfh_test, cfh_test,RTOL) + if __name__ == '__main__': unittest.main() From 440d686ddef89cca18c670c7a08d9a9ff66add52 Mon Sep 17 00:00:00 2001 From: Pedro Amaral Date: Mon, 24 Jun 2024 19:17:04 -0300 Subject: [PATCH 08/10] Update setup in test_sur_error.py --- spreg/tests/test_sur_error.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/spreg/tests/test_sur_error.py b/spreg/tests/test_sur_error.py index 9bcb2961..d592b262 100755 --- a/spreg/tests/test_sur_error.py +++ b/spreg/tests/test_sur_error.py @@ -2,13 +2,15 @@ import numpy as np import libpysal from libpysal.examples import load_example -from ..sur_utils import sur_dictxy -from ..sur_error import SURerrorML, SURerrorGM -from .test_sur import dict_compare +from spreg.sur_utils import sur_dictxy +from spreg.sur_error import SURerrorML, SURerrorGM from libpysal.common import RTOL ATOL = 0.0001 +def dict_compare(actual, desired, rtol, atol=1e-7): + for i in actual.keys(): + np.testing.assert_allclose(actual[i], desired[i], rtol, atol=atol) class Test_SUR_error(unittest.TestCase): def setUp(self): @@ -316,9 +318,7 @@ class Test_SUR_error_gm(unittest.TestCase): def setUp(self): nat = load_example("Natregimes") self.db = libpysal.io.open(nat.get_path("natregimes.dbf"), "r") - self.w = libpysal.weights.Queen.from_shapefile( - libpysal.examples.get_path("natregimes.shp") - ) + self.w = libpysal.weights.Queen.from_shapefile(nat.get_path("natregimes.shp")) self.w.transform = "r" def test_error_gm(self): # 2 equations From ccdfb5159d352d896815c26751ffe2cb244eb709 Mon Sep 17 00:00:00 2001 From: eli knaap Date: Mon, 24 Jun 2024 15:47:55 -0700 Subject: [PATCH 09/10] add dgp --- docs/api.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/api.rst b/docs/api.rst index abd4e60c..6410edbf 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -128,3 +128,27 @@ Diagnostic tests are useful for identifying model fit, sufficiency, and specific spreg.panel_rLMlag spreg.panel_rLMerror spreg.panel_Hausman + +DGP +----------- + +Tools for simulating synthetic data according to different spatial data generating processes + +.. autosummary:: + :toctree: generated/ + + spreg.dgp.make_error + spreg.dgp.make_x + spreg.dgp.make_wx + spreg.dgp.make_xb + spreg.dgp.make_wxg + spreg.dgp.dgp_errproc + spreg.dgp.dgp_ols + spreg.dgp.dgp_slx + spreg.dgp.dgp_sperror + spreg.dgp.dgp_slxerror + spreg.dgp.dgp_lag + spreg.dgp.dgp_spdurbin + spreg.dgp.dgp_lagerr + spreg.dgp.dgp_gns + spreg.dgp.dgp_mes \ No newline at end of file From 78c6d00074cbe70f5e205383debe46c3e7d36d9f Mon Sep 17 00:00:00 2001 From: eli knaap Date: Mon, 24 Jun 2024 15:50:58 -0700 Subject: [PATCH 10/10] tweak --- docs/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index 6410edbf..77c232e1 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -132,7 +132,7 @@ Diagnostic tests are useful for identifying model fit, sufficiency, and specific DGP ----------- -Tools for simulating synthetic data according to different spatial data generating processes +Tools for simulating synthetic data according to data-generating processes implied by different spatial model specifications .. autosummary:: :toctree: generated/