Skip to content

Latest commit

 

History

History
366 lines (294 loc) · 16.5 KB

B_I_EDA_quantiles.md

File metadata and controls

366 lines (294 loc) · 16.5 KB

Seasonal flow timeseries model
from paired flow and meterological/weather record

Exploratory Data Analysis: comparison of precipitation vs flow quantiles

ALR July 2014
Conte Anadromous Fish Research Center

Load libraries, functions, etc

library(lme4)
## Loading required package: Matrix
## Loading required package: Rcpp

library(plyr)
library(ggplot2)
library(reshape2)
library(sp)
library(devtools)
## WARNING: Rtools is required to build R packages, but is not currently installed.
## 
## Please download and install Rtools 3.1 from http://cran.r-project.org/bin/windows/Rtools/ and then run find_rtools().
## 
## Attaching package: 'devtools'
## 
## The following objects are masked from 'package:utils':
## 
##     ?, help
## 
## The following object is masked from 'package:base':
## 
##     system.file

library(xtable)


model_dir<-"C:/ALR/Models/MetToFlow"
model_data_dir<-"C:/ALR/Models_processed_data"


#utility functions, load (source) from my saved gist
source_gist("https://gist.github.com/anarosner/ba285306fc0ce9d812a5", sha1="b25a1b73e02cc2b2d2c590f6c0b2c9c9945fa980")
## Sourcing https://gist.githubusercontent.com/anarosner/ba285306fc0ce9d812a5/raw/48b3efa59d36c7dceacf9c1c0a6fd77ca20bfdb5/util.r

setwd(file.path(model_data_dir,"flow_timeseries"))

load("gages_char_spatial.Rdata")
long_site_no<-gages.char.spatial$site_no[gages.char.spatial$qannual>=25 & gages.char.spatial$TNC_DamCount==0 & gages.char.spatial$OnChannelWaterSqKM<0.5]
length(long_site_no)
## [1] 7

# View(t(gages.char.spatial@data[gages.char.spatial$site_no %in% long_site_no,]))
g<-t(gages.char.spatial@data[gages.char.spatial$site_no %in% long_site_no,])
dimnames(g)[[2]]<-g["site_no",]
print(xtable(g),type="html")
01064400 01073000 01075800 01105585 01105600 01118300 01434025
FEATUREID 9311777 5845058 6730101 5863947 5863773 6140826 4147946
agency_cd USGS USGS USGS USGS USGS USGS USGS
site_no 01064400 01073000 01075800 01105585 01105600 01118300 01434025
station_nm LUCY BROOK NEAR NORTH CONWAY, NH OYSTER RIVER NEAR DURHAM, NH STEVENS BROOK NEAR WENTWORTH, NH TOWN BROOK AT QUINCY, MA OLD SWAMP RIVER NEAR SOUTH WEYMOUTH, MA PENDLETON HILL BROOK NEAR CLARKS FALLS, CT. BISCUIT BK ABOVE PIGEON BK AT FROST VALLEY NY
dec_lat_va 44.07 43.15 43.84 42.25 42.19 41.47 42.00
dec_long_va -71.17 -70.97 -71.89 -71.00 -70.94 -71.83 -74.50
coord_acy_cd S S S S U H 1
dec_coord_datum_cd NAD83 NAD83 NAD83 NAD83 NAD83 NAD83 NAD83
huc_cd 01060002 01060003 01070001 01090001 01090001 01090005 02040104
drain_area_va 4.68 12.10 2.94 4.11 4.50 4.02 3.72
sv_begin_date 1987-10-09 1936-03-19 1987-10-08 1982-12-16 1967-04-18 1958-07-11 1983-02-16
sv_end_date 1992-05-27 2014-06-02 1998-10-01 2014-06-02 2014-06-05 2014-06-25 2014-05-20
sv_count_nu 42 314 98 127 268 482 413
da_sqkm 12.121 31.339 7.615 10.645 11.655 10.412 9.635
qseasonal 112 249 141 105 179 209 111
qannual 27 63 35 27 45 53 28
Forest 97.799 73.528 99.118 8.895 33.360 83.289 100.000
Herbacious 0.6441 5.7393 0.3083 4.6497 0.2096 4.9337 0.0000
Agriculture 0.62107 5.88559 0.33586 0.06358 0.08576 6.15487 0.00000
Developed 0.9354 11.8234 0.2382 79.8569 66.3444 5.3472 0.0000
DevelopedNotOpen 0.1150 3.6475 0.0000 73.7455 50.1991 0.7673 0.0000
Impervious 0.08174 2.16175 0.01724 48.61790 27.67411 0.70356 0.00000
AnnualTmaxC 11.77 14.33 11.39 15.39 15.47 15.44 10.40
AnnualTminC -0.6727 2.4447 -0.1976 5.2691 4.8915 4.5252 0.1656
AnnualPrcpMM 1401 1161 1282 1278 1256 1216 1524
SummerPrcpMM 369.1 292.1 344.6 299.2 289.3 292.3 398.3
WinterPrcpMM 313.9 247.2 274.6 314.1 312.6 283.0 319.4
DrainageClass 1.593 3.347 3.105 3.276 3.919 3.570 2.488
HydrologicGroupAB 87.5000 49.5472 76.6733 45.7568 44.2069 68.9397 0.1498
HydrologicGroupCD 12.50 47.43 22.52 54.24 55.79 31.06 98.35
SurficialCoarseC 48.000 7.842 13.059 15.538 26.931 4.692 0.000
PercentSandy 4.1290 0.2672 9.1447 11.0852 17.1089 0.4179 0.0000
ReachElevationM 295.79000 48.70422 282.19044 0.01083 33.64731 56.35676 727.91603
BasinElevationM 431.60 58.71 486.11 24.05 42.92 90.72 865.67
ReachSlopePCNT 5.2180 0.8770 7.6185 0.0113 0.5419 1.2008 4.9676
BasinSlopePCNT 25.189 4.996 22.682 4.987 3.474 6.912 25.826
TotDASqKM 14.296 32.043 7.866 15.985 11.826 13.701 9.918
TNC_DamCount 0 0 0 0 0 0 0
deg_barr_1 0 0 0 0 0 0 0
deg_barr_2 0 0 0 0 0 0 0
deg_barr_3 0 0 0 0 0 0 0
deg_barr_4 0 0 0 0 0 0 0
deg_barr_6 0 0 0 0 0 0 0
deg_barr_7 0 0 0 0 0 0 0
OnChannelWaterSqKM 0.000000 0.445174 0.000000 0.164880 0.005342 0.033938 0.000000
OnChannelWetlandSqKM 0.1151 2.3047 0.0000 0.1046 0.7245 0.7986 0.0000
OffChannelWaterSqKM 0.000000 0.048012 0.000000 0.036367 0.007902 0.049986 0.000000
OffChannelWetlandSqKM 0.03532 0.90935 0.00272 0.17506 0.91673 0.30900 0.00000
large_barriers 0 0 0 0 0 0 0
small_barriers 0 0 0 0 0 0 0 
load("dseasonal.Rdata")
## Warning: cannot open compressed file 'dseasonal.Rdata', probable reason
## 'No such file or directory'

## Error: cannot open the connection

dseasonal.long<-dseasonal[dseasonal$site_no %in% long_site_no,]
## Error: object 'dseasonal' not found

dseasonal.long<-subset(dseasonal.long,subset=!is.na(precip.e) & !is.na(precip.e.lag1) & !is.na(precip.e.lag2) )
## Error: object 'dseasonal.long' not found

# dseasonal.long[is.na(dseasonal.long$precip.e.lag2),]
# dim(dseasonal.long)



# 1 = Complete barrier to all fish (12+ feet)
# 2 = Small dam barrier (1-12 feet)
# 3 = Partial breach
# 4 = Barrier with fish ladder
# 5 = Unlikely barrier - fully breached, weir, under 1ft dam (also COND=NO DAM or COND=REM or COND=DEL)
# 6 = Unknown, assumed full barriers
# 7 = Locks
basic<-lmer(log(flow) ~ 
              log(da_sqkm) +
               precip.e +  precip.e.lag1 + precip.e.lag2 +    #precip.e.lag3 + 
               (1|year)+(1|site_no),
          na.action=NULL, #to ensure we attached fitted values w/ correct site_no
         data=dseasonal.long) 
## Error: 'data' not found, and some variables missing from formula
## environment

season.names<-c("winter","spring","summer","fall")
# basic.season<-list()
weights<-as.data.frame(matrix(nrow=4,ncol=3))

for (i in 1:length(season.names)) {
     temp<-update(basic, subset=season==season.names[i])
     weights[i,]<-fixef(temp)[c("precip.e", "precip.e.lag1", "precip.e.lag2")]/
          sum(fixef(temp)[c("precip.e", "precip.e.lag1", "precip.e.lag2")])
#      basic.season[season.names[i]]<-temp
}
## Error: error in evaluating the argument 'object' in selecting a method for function 'update': Error: object 'basic' not found

row.names(weights)<-season.names
names(weights)<-c("precip.e", "precip.e.lag1", "precip.e.lag2")

weights
##        precip.e precip.e.lag1 precip.e.lag2
## winter       NA            NA            NA
## spring       NA            NA            NA
## summer       NA            NA            NA
## fall         NA            NA            NA

compare precip vs flow quantiles from sites w long records

# 4 sites, 4 seasons

rm(quant.melt)
for (i in long_site_no) {
#      i<-long_site_no[2]
     print(i)
     for (j in 1:length(season.names)) {
          print(j)
          print(season.names[j])
          temp<-dseasonal.long[dseasonal.long$site_no == i &
                                    dseasonal.long$season==season.names[j],
                            c("precip.e","precip.e.lag1","precip.e.lag2","flow","year")]
          temp.weighted<-temp[,1]*weights[j,1] + temp[,2]*weights[j,2] + temp[,3]*weights[j,3]
          
          cdf.q<-ecdf(temp$flow)
          cdf.p<-ecdf(temp$precip.e)
          cdf.p.weighted<-ecdf(temp.weighted)
          
          temp.q<-data.frame(year=temp$year,
                             flow=cdf.q(temp$flow),
                             precip=cdf.p(temp$precip.e),
                             precip.weighted=cdf.p.weighted(temp.weighted),
                             site=i,
                             season=season.names[j],
                             stringsAsFactors=F)

          temp.q$high<-apply(temp.q[,c("flow","precip")],MARGIN=1,max)
          temp.q$low<-apply(temp.q[,c("flow","precip")],MARGIN=1,min)
          temp.q$high.weighted<-apply(temp.q[,c("flow","precip.weighted")],MARGIN=1,max)
          temp.q$low.weighted<-apply(temp.q[,c("flow","precip.weighted")],MARGIN=1,min)
#           temp.q$under<-(temp.q$flow-temp.q$precip)>0
          head(temp.q)
     
          temp.melt<-melt(temp.q,id.vars=c("year","site", "season","high","low","high.weighted","low.weighted"))
          {     
          if(!exists("quant.melt"))
               quant.melt<-temp.melt
          else{
#                print((nrow(quant.melt)+1))
#                print((nrow(quant.melt)+nrow(temp.melt)))
               quant.melt[(nrow(quant.melt)+1):
                               (nrow(quant.melt)+nrow(temp.melt)),]<-temp.melt
          }
          }
     }
}
## [1] "01064400"
## [1] 1
## [1] "winter"

## Error: object 'dseasonal.long' not found

head(quant.melt)
## Error: error in evaluating the argument 'x' in selecting a method for function 'head': Error: object 'quant.melt' not found

quant.melt$variable<-(capitalize(as.character(quant.melt$variable)))
## Error: object 'quant.melt' not found

quant.melt$season<-capitalize(quant.melt$season)
## Error: object 'quant.melt' not found

# names(quant.melt)[names(quant.melt)=="value"]<-"quantile"

# quant.melt$year.factor<-as.factor(quant.melt$year)
unique(quant.melt$variable)
## Error: object 'quant.melt' not found

gg.quant<-ggplot(data=subset(quant.melt,variable %in% c("Flow","Precip" )),
                 aes(x=year,y=value,color=variable,lty=variable)) +
     theme_bw()+scale_colour_hue(l=40)+ylab("quantile")+theme(legend.title=element_blank())
## Error: object 'quant.melt' not found

limits <- aes(ymax = high, ymin=low)

#      i<-capitalize(season.names[1])
for (i in capitalize(season.names)) {  
     temp<-gg.quant +  
          geom_line(subset=.(season==i),colour="grey50") + 
          geom_errorbar(limits,colour="grey30",lwd=1.1,subset=.(season==i)) +
          geom_point(pch=15,cex=3,subset=.(season==i)) +
          facet_wrap(~site,nrow=2) + ggtitle(i)
     print(temp)
     
     print(
          temp %+% subset(quant.melt,variable %in% c("Flow","Precip.weighted")) +
              ggtitle(paste("Weighted quantiles:",i))
           )
     
#      setwd(file.path(model_dir,"B_eda"))
#      ggsave(temp,filename=paste0("quantile_",i,".png"),width=16,height=8)
#      ggsave(temp,filename=paste0("quantile_weighted_",i,".png"),width=16,height=8)
}
## Error: object 'gg.quant' not found