update for presentation

Author: bpb824

_data/freeway/speedValidation/Copy of speedValidationSheet.numbers

@@ -1,6 +1,6 @@
 # Meta --------------------------------------------------------------------
 # @file missingDataFinder.R
-# @description This script finds missing data within the PORTAL database. 
+# @description This script finds missing data within the PORTAL database.
 # @author Bryan Blanc <bryanpblanc@gmail.com>
 
 # Initialization ----------------------------------------------------------
@@ -36,6 +36,7 @@ source("_functions/aggregation.R")
 missingData = function(station_id,detectors,startTime,endTime,percentile){
   dets = detectors$detectorid[detectors$stationid %in% station_id]
   lanes = sort(unique(detectors$lanenumber[detectors$detectorid %in% dets]))
+  lanes = lanes[lanes!=0]
   laneList = list()
   if(length(dets)>0){
     query = freewayQuery(dets,startTime,endTime)
@@ -125,15 +126,15 @@ colnames(mapFrame)=c("station_id","long","lat","htmlLink","numOutages")
 for(i in 1:length(stationList)){
   sid = as.numeric(names(stationList[i]))
   mapFrame$station_id[i]=sid
-  
+
   if(sid <5000){
     mapFrame$lat[i] = staLocs$Y[staLocs$stationid==sid]
     mapFrame$long[i] = staLocs$X[staLocs$stationid==sid]
   }else{
     mapFrame$lat[i] = staLocs$Y[staLocs$stationid==sid-4000]
     mapFrame$long[i] = staLocs$X[staLocs$stationid==sid-4000]
   }
-  
+
   sta = stationList[[i]]
   numLanes = length(unique(detectors$lanenumber[detectors$stationid==sid]))
   if(numLanes >0){
@@ -154,9 +155,9 @@ for(i in 1:length(stationList)){
     melted$day = as.Date(melted$day)
     flagged = subset(melted,melted$value!=0)
     mapFrame$numOutages[i]=nrow(flagged)
-    
-    
-    
+
+
+
     portalLink=paste0("http://portal.its.pdx.edu/Portal/index.php/stations/view/id/",sid,"/")
     if(sid>=5000){
       mapFrame$htmlLink[i] = paste0("<div align='center'><b>Ramp<b><br><br/><a href='",portalLink,"' target=_blank>",stations$locationtext[stations$stationid==sid][1],"</a></div>")
@@ -175,9 +176,9 @@ pal <- colorNumeric(
   domain = c(0,100)
 )
 
-map= leaflet(mapFrame) %>% addProviderTiles("CartoDB.DarkMatter",options = providerTileOptions(noWrap = TRUE)) %>% 
+map= leaflet(mapFrame) %>% addProviderTiles("CartoDB.DarkMatter",options = providerTileOptions(noWrap = TRUE)) %>%
   setView(-122.6662589, 45.5317385, zoom = 10) %>% addCircleMarkers(lng=~long,lat=~lat,popup =~htmlLink,color =~pal(numOutages))
-  
+
 
 
 # Plotting ----------------------------------------------------------------
@@ -209,7 +210,7 @@ for(i in 1:length(stationList)){
     flagged$id = rownames(flagged)
     plotList[[as.character(sid)]]=flagged
   }
-  
+
   #ggplot(flagged,aes(x = day, y = variable, group = value, colour=value))+geom_point()+scale_x_date()+scale_color_discrete(labels=flagKey$def)
   #flagged %>% ggvis(x=~day, y=~ variable, stroke =~value, fill =~value, key:=~id)  %>% layer_points(opacity := 0.5) %>% add_tooltip(tFunk)
 }
@@ -219,3 +220,48 @@ for(i in 1:length(stationList)){
 saveRDS(plotList,"_data/freeway/dataOutages/plotList.rds")
 
 
+# Table -------------------------------------------------------------------
+flagKey = data.frame(matrix(nrow=4,ncol=2))
+colnames(flagKey)=c("value","def")
+flagKey$value=1:4
+flagKey$def = c("Very Low Values","Only Empty Data", "Zero Rows of Data", "Detector Unavailable")
+stationList = readRDS("_data/freeway/dataOutages/stationList.rds")
+
+tabFrame = data.frame(matrix(nrow=length(stationList),ncol=6))
+colnames(tabFrame)=c("sid","location","flag1","flag2","flag3","numOutages")
+#colnames(tabFrame)=c("station_id",c("# Days w/ Very Low Values","# Days w/ Only Empty Data", "# Days w/ Zero Rows of Data"),"# Days w/ Outages (Total)")
+
+for(i in 1:length(stationList)){
+  sid = as.numeric(names(stationList[i]))
+  tabFrame$sid[i]=sid
+  tabFrame$location[i] = stations$locationtext[stations$stationid==sid][1]
+  sta = stationList[[i]]
+  numLanes = length(unique(detectors$lanenumber[detectors$stationid==sid]))
+  if (numLanes>0){
+    flag1 = 0
+    flag2 = 0
+    flag3 = 0
+    for (j in 1:length(sta)){
+      flags = unlist(sta[[j]])
+      for (k in 1:length(flags)){
+        if(flags[k]==1){
+          flag1 = flag1 +1
+        }else if(flags[k]==2){
+          flag2 = flag2 +1
+        }else if(flags[k]==3){
+          flag3 = flag3 +1
+        }
+      }
+    }
+    tabFrame$flag1[i]=flag1
+    tabFrame$flag2[i]=flag2
+    tabFrame$flag3[i]=flag3
+    tabFrame$numOutages[i]=flag1+flag2+flag3
+  }
+}
+
+tabFrame = tabFrame[complete.cases(tabFrame),]
+
+saveRDS(tabFrame,"_data/freeway/dataOutages/tabFrame.rds")
+
+

_data/freeway/speedValidation/speedValidation2.csv

@@ -5,6 +5,7 @@ require(plyr)
 require(ggplot2)
 require(lubridate)
 require(rjson)
+require(portalr)
 
 setwd("/Users/bblanc/OneDrive/_ODOT/_Portal/investigations")
 

_results/img/speedValidation/I-205 SB @ OR224 Lane 1.png

@@ -4,59 +4,60 @@ library(timeSeries)
 library(plyr)
 library(ggplot2)
 library(lubridate)
+library(rjson)
+library(rgdal)
+require(portalr)
 
-setwd("/Users/bblanc/OneDrive/_ODOT/_Portal/investigations/histBuilder/")
+setwd("/Users/bblanc/OneDrive/_ODOT/_Portal/investigations/")
 
 ####Connect to Portal db
 ####Make sure you VPN into cecs network
-db_cred = fromJSON(file="/Users/bblanc/OneDrive/_ODOT/_Portal/investigations/db_credentials.json")
+db_cred = fromJSON(file="db_credentials.json")
 con <- dbConnect(dbDriver("PostgreSQL"), host=db_cred$db_credentials$db_host, port= 5432, user=db_cred$db_credentials$db_user, password = db_cred$db_credentials$db_pass, dbname=db_cred$db_credentials$db_name)
 
-
 ####Relational Tables
 stations= dbGetQuery(con,"SELECT * FROM public.stations")
 highways = dbGetQuery(con,"SELECT * FROM public.highways")
 detectors = dbGetQuery(con,"SELECT * FROM public.detectors")
 corridors  = dbGetQuery(con,"SELECT * FROM public.corridors")
 corridor_stations = dbGetQuery(con,"SELECT * FROM public.corridorstations")
+#
+# ##Load standard PORTAL functions
+# source("/Users/bblanc/OneDrive/_ODOT/_Portal/investigations/_codeResources/portalFxns.R")
 
-##Load standard PORTAL functions
-source("/Users/bblanc/OneDrive/_ODOT/_Portal/investigations/_codeResources/portalFxns.R")
+oregonStationGeom = readOGR("_data/GIS/","oregonStations")
+orStationData = oregonStationGeom@data
+currentStations = subset(orStationData,is.na(orStationData$end_date))
+stationVec = sort(unique(currentStations$stationid))
 
-juneRaw = readRDS("juneData.rds")
+juneRaw = readRDS("_data/freeway/juneData.rds")
 
 juneAgg= list()
 for (i in 1:length(juneRaw)){
   sid = juneRaw[[i]]$station_id
-  sData = juneRaw[[i]]$data
-  if(nrow(sData)>0){
-    #clean = filter(sData)
-    #if(nrow(clean)>0){
-      clean = join(sData,detectors,by="detectorid")
-      clean$period = cut(clean$starttime,breaks="1 hour")
-      agg = ddply(clean,c("lanenumber","period"),function(X) data.frame(volume=sum(X$volume), occupancy = mean(X$occupancy), speed = weighted.mean(X$speed,X$volume)))
-      agg$period=as.POSIXct(agg$period)
-      agg$dow = weekdays(agg$period,abbreviate=TRUE)
-      agg$hour = hour(agg$period)
-      agg_hd = ddply(agg,c("lanenumber","hour","dow"),function(X) data.frame(volume=mean(X$volume), occupancy = mean(X$occupancy), speed = mean(X$speed)))
-      station = list()
-      station[["station_id"]]=sid
-      station[["data"]]=agg_hd
-      juneAgg[[i]]=station
-    #}else{
-    #  station = list()
-    #  station[["station_id"]]=sid
-    #  station[["data"]]=NA
-    #  juneAgg[[i]] = station
-    #}
-    
-  }else{
-    station = list()
-    station[["station_id"]]=sid
-    station[["data"]]=NA
-    juneAgg[[i]] = station
+  if(sid %in% stationVec){
+    sData = juneRaw[[i]]$data
+    if(nrow(sData)>0){
+      clean = portalr::filterFreeway(sData)
+      if(nrow(clean)>0){
+        clean = join(sData,detectors,by="detectorid")
+        clean$period = cut(clean$starttime,breaks="1 hour")
+        agg = ddply(clean,c("lanenumber","period"),function(X) data.frame(volume=sum(X$volume), occupancy = mean(X$occupancy), speed = weighted.mean(X$speed,X$volume)))
+        agg$period=as.POSIXct(agg$period)
+        agg$dow = weekdays(agg$period,abbreviate=TRUE)
+        agg$hour = hour(agg$period)
+        agg_hd = ddply(agg,c("lanenumber","hour","dow"),function(X) data.frame(volume=mean(X$volume), occupancy = mean(X$occupancy), speed = mean(X$speed)))
+        juneAgg[[as.character(sid)]] = agg_hd
+      }else{
+        juneAgg[[as.character(sid)]] = NA
+      }
+
+    }else{
+      juneAgg[[as.character(sid)]] = NA
+    }
   }
   print (i)
 }
 
-saveRDS(juneAgg,"juneAggData.rds")
+saveRDS(juneAgg,"_data/freeway/juneAggData_clean.rds")
+saveRDS(currentStations, "_data/freeway/stationMeta.rds")

_results/img/speedValidation/I-205 SB @ OR224 Lane 3.png

@@ -1,4 +1,4 @@
-big# Initialization ----------------------------------------------------------
+# Initialization ----------------------------------------------------------
 
 #Load neccesary libraries
 require(RPostgreSQL)
@@ -90,9 +90,9 @@ dev.off()
 
 # Issue 6: I-205 SB @ OR-224/82nd: Missing Data -----------------------------------------------------------------
 stas = 1099
-subStations = subset(stations,stations$stationid %in% stas)
 dets = detectors$detectorid[detectors$stationid %in% stas ]
 subDetectors = subset(detectors,detectors$detectorid %in% dets)
+head(subDetectors)
 startDate = "2015-05-01"
 endDate = "2015-05-31"
 raw = dbGetQuery(con,freewayQuery(dets,startDate,endDate))

_results/img/speedValidation/US26 WB @ 185th lane 2.png

@@ -1,3 +1,4 @@
+require(portalr)
 require(RPostgreSQL)
 require(DBI)
 require(timeSeries)
@@ -6,6 +7,7 @@ require(ggplot2)
 require(lubridate)
 require(reshape2)
 require(rjson)
+require(portalr)
 
 setwd("/Users/bblanc/OneDrive/_ODOT/_Portal/investigations/")
 
@@ -16,14 +18,9 @@ con <- dbConnect(dbDriver("PostgreSQL"), host=db_cred$db_credentials$db_host, po
 
 ####Relational Tables
 stations= dbGetQuery(con,"SELECT * FROM public.stations")
-highways = dbGetQuery(con,"SELECT * FROM public.highways")
 detectors = dbGetQuery(con,"SELECT * FROM public.detectors")
-corridors  = dbGetQuery(con,"SELECT * FROM public.corridors")
-corridor_stations = dbGetQuery(con,"SELECT * FROM public.corridorstations")
-
-###Load Querying Functions
-source("_functions/portalFxns.R")
 
+#First round
 speeds = read.csv("_data/freeway/speedValidation/speedValidation.csv",stringsAsFactors = FALSE)
 speeds = speeds[,1:3]
 speeds$time = paste0("2015/08/19 ",speeds$Timestamp)
@@ -77,6 +74,43 @@ speedList[[8]]$crossStreet ="33rd Ave"
 speedList[[8]]$lane=1
 speedList[[8]]$sid = 1062
 
+#Second Round
+speeds = read.csv("_data/freeway/speedValidation/speedValidation2.csv",stringsAsFactors = FALSE)
+speeds = speeds[,1:3]
+speeds$time = paste0("2015/09/03 ",speeds$Timestamp)
+speeds$time = as.POSIXct(strptime(speeds$time,format = "%Y/%m/%d %H:%M:%S"))
+
+locs = unique(speeds$Location)
+locs = locs[locs != ""]
+
+indices = which(speeds$Location %in% locs)
+indices[5]=nrow(speeds)
+
+for (i in 1:(length(locs))){
+  subList = list()
+  subList[["location"]]=locs[i]
+  sub = speeds[(indices[i]:(indices[i+1]-1)),c(2,4)]
+  subList[["observed"]]=sub[complete.cases(sub),]
+  speedList[[i+8]]=subList
+}
+
+speedList[[9]]$highway="US-26 WB"
+speedList[[9]]$crossStreet ="185th Ave."
+speedList[[9]]$lane=1
+speedList[[9]]$sid = 1129
+speedList[[10]]$highway="US-26 WB"
+speedList[[10]]$crossStreet ="185th Ave."
+speedList[[10]]$lane=2
+speedList[[10]]$sid = 1129
+speedList[[11]]$highway="I-205 SB"
+speedList[[11]]$crossStreet ="Clackamas Hwy"
+speedList[[11]]$lane=1
+speedList[[11]]$sid = 1100
+speedList[[12]]$highway="I-205 SB"
+speedList[[12]]$crossStreet ="Clackamas Hwy"
+speedList[[12]]$lane=3
+speedList[[12]]$sid = 1100
+
 saveRDS(speedList,"_data/freeway/speedValidation/speedList.rds")
 speedList=readRDS("_data/freeway/speedValidation/speedList.rds")
 
@@ -94,32 +128,37 @@ for(i in 1:length(speedList)){
   start = as.character(round(start,"mins"))
   end = max(obs$time)
   end = as.character(round(end,"mins"))
-  
-  request = dbGetQuery(con,freewayQuery_time(did,start,end))
+
+  request = freewayData(con,did,start,end)
   speedList[[i]]$portal = request
-  
+
   histStart = as.POSIXct(start)
   week(histStart)<-week(histStart)-9
   histStart = as.character(histStart)
   histStartSeq = as.character(timeSequence(from = histStart, to = start, by = "week"))
-  
+
   histEnd = as.POSIXct(end)
   week(histEnd)<-week(histEnd)-9
   histEnd = as.character(histEnd)
   histEndSeq = as.character(timeSequence(from = histEnd, to = end, by = "week"))
-  
+
   query = paste0("SELECT * FROM freeway.data WHERE ((starttime >='",histStartSeq[1],"' AND starttime <='",histEndSeq[1],"')")
   for (j in 2:length(histStartSeq)){
     query = paste0(query," OR (starttime >='",histStartSeq[j],"' AND starttime <='",histEndSeq[j],"')")
   }
   query = paste0(query,") AND detectorid = ",did)
-  
+
   request = dbGetQuery(con,query)
   request$time = paste0(hour(request$starttime),":",minute(request$starttime),":",second(request$starttime))
-  
+
   agg = ddply(request,c("time"),summarise,speed=mean(speed,na.rm=TRUE))
-  agg$time = as.POSIXct(strptime(paste0("2015-08-19 ",agg$time),format= "%Y-%m-%d %H:%M:%S"))
+  if(i %in% 1:8){
+    agg$time = as.POSIXct(strptime(paste0("2015-08-19 ",agg$time),format= "%Y-%m-%d %H:%M:%S"))
+  }else{
+    agg$time = as.POSIXct(strptime(paste0("2015-09-03 ",agg$time),format= "%Y-%m-%d %H:%M:%S"))
+  }
 
+
   speedList[[i]]$hist = agg
 }
 
@@ -129,28 +168,28 @@ for (i in 1:length(speedList)){
   obs = speedList[[i]]$observed
   obs$period = cut(obs$time,"1 min")
   obsAgg = ddply(obs,c("period"),summarise,speed = mean(Reading))
-  
+
   portal = speedList[[i]]$portal
   portal$period = cut(portal$starttime,"1 min")
   portalAgg = ddply(portal,c("period"),summarise,speed = mean(speed))
-  
+
   hist = speedList[[i]]$hist
   hist$period = cut(hist$time,"1 min")
   histAgg = ddply(hist,c("period"),summarise,speed = mean(speed))
-  
+
   joined = join(obsAgg,portalAgg,by="period")
   joined = join(joined,histAgg, by ="period")
   colnames(joined)=c("time","observed","portal","hist")
   melted = melt(joined,id="time")
   melted$time=as.POSIXct(melted$time)
-  
+
   speedList[[i]]$melted = melted
-  
-  
+
+
   plot=ggplot(melted,aes(x=time,y=value,group=variable,colour=variable))+geom_smooth()+scale_x_datetime()+
     ggtitle(paste0(speedList[[i]]$highway," @ ",speedList[[i]]$crossStreet,", Lane",speedList[[i]]$lane))+
     ylab("Speed (mph)")+xlab("")
-  
+
   png(paste0("_results/img/speedValidation/",speedList[[i]]$location,".png"),width=900,height=600)
   print(plot)
   dev.off()

_results/img/speedValidation/US26 WB @185 lane 1.png

@@ -24,25 +24,25 @@ shinyApp(
       )
     )
   ),
-  
+
   server = function(input,output,session){
-    
+
     hplot = reactive({
       hourSelect = input$hour
       binSelect = input$bin
-      
+
       sub = subset(blue,blue$hour==hourSelect)
-      
+
       percentBins = quantile(sub$tt_min,probs = seq(0,1.0,0.05))
       binNums = data.frame(matrix(nrow=length(percentBins),ncol=2))
       colnames(binNums)=c("name","num")
       binNums$name[1:20]= paste0(names(percentBins)[1:20],"-",names(percentBins)[2:21])
       binNums$name[21]= "100%"
       binNums$num=as.numeric(rownames(binNums))
-      
+
       for (i in 1:nrow(sub)){
         obs = sub$tt_min[i]
-        
+
         if(obs==max(percentBins[length(percentBins)])){
           binNames = "95%-100%"
         }else if(obs ==min(percentBins[1])){
@@ -53,31 +53,31 @@ shinyApp(
           upper = names(percentBins[percentBins == min(percentBins[obs <percentBins])])
           binNames = paste0(lower,"-",upper)
         }
-        
+
         sub$pBin[i]= binNames
         sub$pBinNum[i] = binNums$num[binNums$name == binNames]
       }
-      
+
       sub$pBin=factor(sub$pBin)
-      
+
       tFunk = function(df){
         paste0("Percentile: ",binNums$name[df$pBinNum])
       }
-      
+
       colRamp = colorRampPalette(colors=c("white","red"))
-      
+
       h <- sub %>% ggvis(x = ~tt_min, fill=~pBinNum) %>%
         scale_numeric("fill",label="Percentile Bin", domain=c(1,20),range =colRamp(20)[c(1,20)])%>% group_by(pBinNum)  %>%
         layer_histograms(width=binSelect) %>%
         add_axis("x", title = "Travel Time (minutes)")%>%
         add_axis("y", title = "Frequency") %>% add_tooltip(tFunk)
-      
+
       return(h)
-      
+
     })
-    
+
     hplot %>% bind_shiny("ggvis", "ggvis_ui")
-    
+
   }
 )
 
@@ -98,7 +98,35 @@ tt_filter= function(data,dist){
   minFilter = floor((dist/(35+15))*3600)
   upper = mean(data$traveltime)+1.65*sd(data$traveltime)
   data = subset(data,data$traveltime >=minFilter & data$traveltime <=maxFilter & data$traveltime <= upper)
-  
+
 }
 
+require(dplyr)
+setwd("~/_ODOT_Portal/investigations")
+####Connect to Portal db
+####Make sure you VPN into cecs network
+db_cred = fromJSON(file="/Users/bblanc/OneDrive/_ODOT/_Portal/investigations/db_credentials.json")
+con <- dbConnect(dbDriver("PostgreSQL"), host=db_cred$db_credentials$db_host, port= 5432, user=db_cred$db_credentials$db_user, password = db_cred$db_credentials$db_pass, dbname=db_cred$db_credentials$db_name)
+
+db = src_postgres(dbname = db_cred$db_credentials$db_name,
+                  host = db_cred$db_credentials$db_host,
+                  port = 5432,
+                  user = db_cred$db_credentials$db_user,
+                  password = db_cred$db_credentials$db_pass,
+                  options = "-c search_path=odot")
+
+tbls = src_tbls(db)
+
+ttseg_tbls = tbls[grep("ttseginventory",tbls)]
+ttseg_tblRef = data.frame(matrix(nrow=length(ttseg_tbls),ncol=2))
+colnames(ttseg_tblRef)=c("tbl_name","date")
+ttseg_tblRef$tbl_name=ttseg_tbls
+ttseg_tblRef$date = as.Date(paste0(substr(ttseg_tblRef$tbl_name,16,19),"-",
+                                   substr(ttseg_tblRef$tbl_name,20,21),"-",
+                                   substr(ttseg_tblRef$tbl_name,22,23)))
+
+trav_tbls = tbls[grep("ttdcutraversals",tbls)]
+
+test = tbl(db,"ttseginventory_20150615")
+testdf = data_frame(tbl)
 

_scripts/dataOutages/dataOutage.R

@@ -1,51 +1,75 @@
 library(shiny)
 library(leaflet)
 library(RColorBrewer)
+library(DT)
 
 plotList = readRDS("plotList.rds")
 mapFrame = readRDS("mapFrame.rds")
+tabFrame = readRDS("tabFrame.rds")
 
 timePoints = as.Date(c("2015-01-01","2015-07-31"))
 
 function(input, output, session) {
-  
+
   pal <- colorNumeric(
     palette = c("Green","Yellow","Red"),
     domain = c(0,max(mapFrame$numOutages))
   )
-  
+
   output$leaf <- renderLeaflet({
-    leaflet(mapFrame) %>% addProviderTiles("CartoDB.DarkMatter",options = providerTileOptions(noWrap = TRUE)) %>% 
+    leaflet(mapFrame) %>% addProviderTiles("CartoDB.DarkMatter",options = providerTileOptions(noWrap = TRUE)) %>%
       setView(-122.6662589, 45.5317385, zoom = 9) %>% addCircleMarkers(lng=~long,lat=~lat,popup =~htmlLink,color =~pal(numOutages),fillOpacity = 0.6,stroke=FALSE,layerId=~station_id) %>%
       addLegend("bottomright", pal = pal, values = ~numOutages,
                 title = "Number of Outages",
                 opacity = 1
       )
   })
-  
+
+  add_title <- function(vis, ..., x_lab = "X units", title = "Plot Title")
+  {
+    add_axis(vis, "x", orient = "top", ticks = 0, title = title,
+             properties = axis_props(
+               axis = list(stroke = "white"),
+               labels = list(fontSize = 0)
+             ), ...)
+  }
+
   plt = reactive({
     markerID = input$leaf_marker_click$id
     if(!is.null(markerID)){
       sid = input$leaf_marker_click$id
       plotFrame = plotList[[as.character(sid)]]
-      
+
       tFunk = function(x){
         if(is.null(x)) return(NULL)
         return(as.character(plotFrame$day[plotFrame$id == x$id]))
       }
-      
+
       plotFrame %>% ggvis(x=~day, y=~ variable, stroke =~def, fill =~def, key:=~id)  %>%
-        layer_points(opacity := 0.5) %>% add_tooltip(tFunk) %>% 
+        layer_points(opacity := 0.5) %>% add_tooltip(tFunk) %>%
         set_options(height = 200, width = "100%") %>%
-          scale_nominal(property="fill", label = "Outage Flag", domain = c("Very Low Values","Only Empty Data", "Zero Rows of Data", "Detector Unavailable"), range = brewer.pal(4,"Set1")) %>% 
+          scale_nominal(property="fill", label = "Outage Flag", domain = c("Very Low Values","Only Empty Data", "Zero Rows of Data", "Detector Unavailable"), range = brewer.pal(4,"Set1")) %>%
         scale_nominal(property="stroke", label = "Outage Flag", domain = c("Very Low Values","Only Empty Data", "Zero Rows of Data", "Detector Unavailable"), range = brewer.pal(4,"Set1")) %>%
         add_axis("y", title = "Lane Number", title_offset=50)%>%
-        add_axis("x", title = "")%>% scale_datetime(property="x",domain = timePoints,nice="month", expand=0)
+        add_axis("x", title = "")%>% scale_datetime(property="x",domain = timePoints,nice="month", expand=0)%>% add_title(title = tabFrame$location[tabFrame$sid == sid])
     }
   })
-  
+
   observeEvent(input$leaf_marker_click,{
     plt %>% bind_shiny("ggvis", "ggvis_ui")
   })
-  
-}
+
+  output$outageData =  renderDataTable({
+    colnames(tabFrame)=c("Station","Location",c("# Days w/ Very Low Values","# Days w/ Only Empty Data", "# Days w/ Zero Rows of Data"),"# Days w/ Outages (Total)")
+
+    createLink <- function(val) {
+      sprintf('<a href="http://portal.its.pdx.edu/Portal/index.php/stations/view/id/%s/" target="_blank" class="btn btn-primary">%s</a>',val,val)
+    }
+
+    links = createLink(tabFrame$Station)
+
+    tabFrame$Station = links
+    datatable(tabFrame,escape = FALSE, options = list(lengthMenu = c(5, 10, 50), pageLength = 5))
+  })
+
+}

_scripts/outlierExploration/dataPull.R

@@ -1,14 +1,23 @@
 library(shiny)
 library(leaflet)
 library(ggvis)
+library(DT)
 
 fluidPage(
   titlePanel(title = "",windowTitle = "Data Outage Explorer"),
-  fixedRow(
-    leafletOutput("leaf", width = "100%",height = 400)
+  tabsetPanel(
+    tabPanel("Outage Explorer",
+             fixedRow(
+               leafletOutput("leaf", width = "100%",height = 300)
+             ),
+             p("Click a point on the map to plot data outages"),
+             fixedRow(
+               uiOutput("ggvis_ui"),ggvisOutput("ggvis")
+             )
     ),
-  p("Click a point on the map to plot data outages"),
-  fixedRow(
-    uiOutput("ggvis_ui"),ggvisOutput("ggvis")
+    tabPanel("Outage Table",
+      dataTableOutput("outageData")
+    )
   )
+  
 )

_scripts/outlierExploration/histPlotter.R

@@ -4,11 +4,6 @@ library(knitr)
 library(stats)
 
 shinyServer(function(input, output, session) {
-  img.width <- 800
-  img.height <- 400
-  options(RCHART_HEIGHT = img.height, RCHART_WIDTH = img.width)
-  opts_chunk$set(fig.width=6, fig.height=4)
-  
   laneCheck = readRDS("laneNumberCheck.rds")
   laneCheck$flag=FALSE
   laneCheck$flag[laneCheck$pDiff_10 <0]=TRUE
@@ -33,17 +28,16 @@ shinyServer(function(input, output, session) {
     }
   }
 
-  h <- laneFrame %>% ggvis(x = ~pDiff_10,fill= ~out) %>% group_by(out) %>%
-    layer_histograms(width=input_slider(min=1,max=10,step=1,value=5,label="Bin Width (mph)")) %>%
-    scale_nominal("fill", label = "Outlier Status",
-                  domain = c("OK", "Minor", "Major"),
-                  range = c("green", "yellow", "red")) %>%
-    set_options(width = img.width, height = img.height) %>% 
-    add_tooltip(tooltipFunc(laneFrame$stationid, laneFrame$pDiff_10,laneFrame$out), 'hover') %>%
-    add_axis("x", title = "5th Percentile Difference in Speed Between Lanes 1 and 2 (mph)")%>%
-    add_axis("y", title = "Frequency")%>%
-    bind_shiny("ggvis", "ggvis_ui")
-    
-  
+  h = reactive({
+    laneFrame %>% ggvis(x = ~pDiff_10,fill= ~out) %>% group_by(out) %>%
+      layer_histograms(width=input$bin) %>%
+      scale_nominal("fill", label = "Outlier Status",
+                    domain = c("OK", "Minor", "Major"),
+                    range = c("green", "yellow", "red")) %>%
+      add_tooltip(tooltipFunc(laneFrame$stationid, laneFrame$pDiff_10,laneFrame$out), 'hover') %>%
+      add_axis("x", title = "5th Percentile Difference in Speed Between Lanes 1 and 2 (mph)")%>%
+      add_axis("y", title = "Frequency") %>% set_options(height = 300, width = "100%")
+  }) 
+  h %>% bind_shiny("ggvis", "ggvis_ui")
 })
 

_scripts/specificIssues/portal_issues.R

@@ -3,12 +3,14 @@ library(ggvis)
 library(knitr)
 
 shinyUI(fluidPage(
-  mainPanel(
-    uiOutput("ggvis_ui"),
-    ggvisOutput("ggvis"),
-    tags$ul(tags$li("Minor outliers lie outside the 'inner fence' with is defined by  the interval [1st Quartile - 1.5*IQR, 3rd Quartile +1.5*IQR]"),
-            tags$li("Major outliers lie outside the 'outer fence' with is defined by the interval [1st Quartile - 3*IQR, 3rd Quartile +3*IQR]"))
-  )
-  
-  
+  fixedRow(column(width = 8,
+    wellPanel(sliderInput("bin","Bin Width (mph)",min=1,max=10,value=5,step=1))
+    )),
+  fixedRow(column(width=8, 
+                  uiOutput("ggvis_ui"),
+                  ggvisOutput("ggvis"),
+                  wellPanel(tags$ul(tags$li("Minor outliers lie outside the 'inner fence' with is defined by  the interval [1st Quartile - 1.5*IQR, 3rd Quartile +1.5*IQR]"),
+                          tags$li("Major outliers lie outside the 'outer fence' with is defined by the interval [1st Quartile - 3*IQR, 3rd Quartile +3*IQR]"))      
+                  )))
+
 ))

_scripts/speedValidation/speedValid.R

@@ -3,22 +3,23 @@ library(ggvis)
 library(knitr)
 library(stats)
 library(DT)
+require(plyr)
 
 shinyServer(function(input, output, session) {
   img.width <- 700
   img.height <- 400
   options(RCHART_HEIGHT = img.height, RCHART_WIDTH = img.width)
   opts_chunk$set(fig.width=7, fig.height=4)
-  
+
   unfiltered = readRDS("juneAggData_raw.rds")
-  filtered = readRDS("juneAggData.rds")
+  filtered = readRDS("juneAggData_clean.rds")
   data = filtered
   stationMeta = readRDS("stations.rds")
   lane = "all"
   quant = "speed"
   hour = 8
   weekdays = c("Mon","Tue","Wed","Thu","Fri","Sat","Sun")
-  
+
   observeEvent(input$quant,({
     quant = tolower(input$quant)
     if(quant=="speed"){
@@ -29,7 +30,7 @@ shinyServer(function(input, output, session) {
     }else if(quant == "volume"){
       binRange = c(100,1000)
       binInc = 100
-      histStart = 500
+      histStart = 200
       binLabel = "Bin Width (vph)"
     }else if (quant=="occupancy"){
       binRange = c(1,10)
@@ -39,7 +40,7 @@ shinyServer(function(input, output, session) {
     }
     updateSliderInput(session = session, inputId = "hist",label =binLabel,value = histStart,min = binRange[1],max=binRange[2],step=binInc)
   }))
-  
+
   plotFrame = reactive({
     laneSelect = input$lane
     if(laneSelect=="All (Aggregated)"){
@@ -48,40 +49,41 @@ shinyServer(function(input, output, session) {
       lane <<- laneSelect
     }
     # print(lane)
-    
+
     if(input$filter=="Unfiltered"){
       data = unfiltered
     }else{
       data = filtered
     }
-    
+
     hour <<-input$hour
     weekdays <<- input$weekdays
-    
+
     quant <<- tolower(input$quant)
     #print(quant)
     #print(weekdays)
     #print(hour)
     #print(lane)
-    
+
     pf = data.frame(matrix(nrow = length(data),ncol = 3))
     colnames(pf)=c("station_id","quantity","out")
     if(lane=="all"){
       for (i in 1:length(data)){
-        if(!is.null(data[[i]])){
-          sid = data[[i]]$station_id
+        if(!is.na(data[i])){
+          sid = as.numeric(names(data[i]))
           pf$station_id[i]=sid
-          sData = data[[i]]$data
+          sData = data[[i]]
           if(is.data.frame(sData)){
             rows = sData$hour==hour & sData$dow %in% weekdays
             if(sum(rows>0)){
               sub = sData[rows,]
               if(quant == "volume"){
-                q = sum(sub$volume)
+                temp = ddply(sub,"lanenumber",summarise,volume = sum(volume))
+                q = mean(temp$volume, na.rm = TRUE)
               }else if (quant =="occupancy"){
-                q = mean(sub$occupancy)
+                q = mean(sub$occupancy, na.rm = TRUE)
               }else if (quant == "speed"){
-                q = weighted.mean(sub$speed,sub$volume)
+                q = weighted.mean(sub$speed,sub$volume, na.rm = TRUE)
               }
               pf$quantity[i]=q
             }else{
@@ -96,7 +98,7 @@ shinyServer(function(input, output, session) {
         }
       }
     }
-#     
+#
 #     else if(lane=="On-Ramp"){
 #       for (i in 1:length(data)){
 #         if(!is.null(data[[i]])){
@@ -133,20 +135,20 @@ shinyServer(function(input, output, session) {
 #     }
     else{
       for (i in 1:length(data)){
-        if(!is.null(data[[i]])){
-          sid = data[[i]]$station_id
+        if(!is.na(data[i])){
+          sid = as.numeric(names(data[i]))
           pf$station_id[i]=sid
-          sData = data[[i]]$data
+          sData = data[[i]]
           if(is.data.frame(sData)){
             rows = sData$hour==hour & sData$dow %in% weekdays & as.character(sData$lanenumber)==lane
             if(sum(rows)>1){
               sub = sData[rows,]
                 if(quant == "volume"){
-                  q = mean(sub$volume)
+                  q = mean(sub$volume, na.rm = TRUE)
                 }else if (quant =="occupancy"){
-                  q = mean(sub$occupancy)
+                  q = mean(sub$occupancy, na.rm = TRUE)
                 }else if (quant == "speed"){
-                  q = mean(sub$speed)
+                  q = mean(sub$speed, na.rm = TRUE)
                 }
                 pf$quantity[i]=q
             }else{
@@ -159,12 +161,12 @@ shinyServer(function(input, output, session) {
           pf$station_id[i]=NA
           pf$quantity[i]=NA
         }
-        
+
       }
     }
     pf$out = NA
     comp = pf[complete.cases(pf[,1:2]),]
-    
+
     if(nrow(comp)>0){
       quantiles = quantile(comp$quantity,c(0.25,0.75),na.rm=TRUE)
       iqr = as.numeric(quantiles[2]-quantiles[1])
@@ -174,64 +176,64 @@ shinyServer(function(input, output, session) {
       comp$out[(comp$quantity <fences[1])|(comp$quantity >fences[4])]="Major"
       comp$out=factor(comp$out,ordered = TRUE)
     }
-    
+
     return(comp)
   })
-  
+
   plt = reactive({
     quant = input$quant
-    
+
     #print(quant)
-    
+
     if(quant=="Speed"){
       xAxis = "Speed (mph)"
-      
+
     }else if(quant == "Volume"){
       xAxis = "Volume (vph)"
     }else if (quant=="Occupancy"){
       xAxis = "Occupancy (% of time)"
     }
-    
+
     h <- plotFrame %>% ggvis(x = ~quantity,fill= ~out) %>% group_by(out) %>%
-      #layer_histograms(width=input$hist) %>%
+      layer_histograms(width=input$hist) %>%
       scale_nominal("fill", label = "Outlier Status",
                     domain = c("OK", "Minor", "Major"),
                     range = c("green", "yellow", "red")) %>%
-      set_options(width = img.width, height = img.height) %>% 
+      set_options(width = img.width, height = img.height) %>%
       add_axis("x", title = xAxis)%>%
-      add_axis("y", title = "Frequency") 
-    
+      add_axis("y", title = "Frequency")
+
     return(h)
   })
-  
+
   densityPlot = reactive({
     quant = input$quant
-    
+
     #print(quant)
-    
+
     if(quant=="Speed"){
       xAxis = "Speed (mph)"
-      
+
     }else if(quant == "Volume"){
       xAxis = "Volume (vph)"
     }else if (quant=="Occupancy"){
       xAxis = "Occupancy (% of time)"
     }
-    
+
     h <- plotFrame %>% ggvis(x = ~quantity) %>%
-      set_options(width = img.width, height = img.height) %>% 
+      set_options(width = img.width, height = img.height) %>%
       add_axis("x", title = xAxis)%>%
       add_axis("y", title = "Probability Density") %>%
       layer_densities(
         adjust = input$bandwidth,kernel="gaussian")
-    
+
     return(h)
   })
-  
+
   plt %>% bind_shiny("ggvis", "ggvis_ui")
-  
+
   densityPlot %>% bind_shiny("ggvis_d", "ggvis_ui_d")
-  
+
   output$mytable =  renderDataTable({
     df = plotFrame()
     quantity = input$quant
@@ -247,18 +249,18 @@ shinyServer(function(input, output, session) {
     for (i in 1:nrow(df)){
       locations[i]= stationMeta$locationtext[stationMeta$stationid==df$`Station ID`[i]][1]
     }
-    
+
     createLink <- function(val) {
       sprintf('<a href="http://portal.its.pdx.edu/Portal/index.php/stations/view/id/%s/" target="_blank" class="btn btn-primary">%s</a>',val,val)
     }
-    
+
     links = createLink(df$`Station ID`)
-    
+
     outTable = cbind(links,locations,df[,2:3])
     colnames(outTable)= c("Station ID","Location",qLab,"Outlier Status")
     datatable(outTable,escape = FALSE) %>% formatStyle("Outlier Status",backgroundColor = styleEqual(c("OK","Minor","Major"),c("white","yellow","red")))
     })
-  
+
   output$sumTable = renderDataTable({
     df = plotFrame()
     statSum = as.data.frame(as.matrix(summary(df$quantity)))
@@ -274,6 +276,6 @@ shinyServer(function(input, output, session) {
     colnames(final)=c("Statistic","Value")
     datatable(final) %>% formatStyle("Statistic",backgroundColor = styleEqual(c("Lower Outer Fence","Lower Inner Fence","Upper Inner Fence","Upper Outer Fence"),c("red","yellow","yellow","red")))
   })
-  
+
 })
 

_shiny/bluetooth_tt/blue.R

@@ -0,0 +1 @@
+load("/Users/bblanc/OneDrive/_ODOT/_Portal/investigations/_shiny/speedValidation/processedSpeedData.rds")

_shiny/dataOutages/server.R

@@ -17,6 +17,11 @@ dataLabs$label = c("Observed","Same-time PORTAL Data","10-Week Historical Averag
 
 shinyServer(function(input,output,session){
 
+  output$legend <- renderImage({
+    # Return a list containing the filename
+    list(src = "legend.png")
+  }, deleteFile = FALSE)
+  
   extract = function(index,agg){
     speed = speeds[[index]]
     data = speed$melted
@@ -50,7 +55,7 @@ shinyServer(function(input,output,session){
     dmn = c(min(ranges),max(ranges))
 
     p = agged %>% ggvis(x=~time,y=~value, fill=~label,stroke=~label) %>% group_by(variable) %>% layer_smooths(se=TRUE) %>%
-      layer_points() %>% add_axis("y",title="Speed (mph)")  %>% add_legend(scales = c("stroke","fill"),title="Data Source") %>%
+      layer_points() %>% add_axis("y",title="Speed (mph)")  %>% hide_legend(scales = c("stroke","fill")) %>%
       set_options(height = 400, width = "100%") %>% scale_numeric("y",domain= dmn, nice = TRUE)%>% add_title(title = lane, x_lab = "Time")
       #%>% #add_tooltip(tFunk)
 
@@ -69,7 +74,7 @@ shinyServer(function(input,output,session){
     dmn = c(min(ranges),max(ranges))
 
     p = agged %>% ggvis(x=~time,y=~value, fill=~label,stroke=~label) %>% group_by(variable) %>% layer_smooths(se=TRUE) %>%
-      layer_points() %>% add_axis("y",title="Speed (mph)") %>% add_legend(scales = c("stroke","fill"),title="Data Source") %>%
+      layer_points() %>% add_axis("y",title="Speed (mph)") %>% hide_legend(scales = c("stroke","fill")) %>%
       set_options(height = 400, width = "100%") %>% scale_numeric("y",domain= dmn, nice = TRUE)%>%add_title(title = lane, x_lab = "Time")
 
     return(p)

_shiny/dataOutages/ui.R

@@ -14,7 +14,10 @@ locVec = unique(locVec)
 
 shinyUI(
   fluidPage(
-    titlePanel(title="Speed Validation Explorer"),
+    fixedRow(
+      column(8,h1("Speed Validation Explorer")),
+      column(4,imageOutput("legend", height ="10px",inline = TRUE))
+    ),
     fixedRow(column(6,
                     uiOutput("ggvis_ui_1"),ggvisOutput("ggvis_1")
                     ),
@@ -24,7 +27,7 @@ shinyUI(
     ),
     fixedRow(column(12,
                     wellPanel(selectInput("loc","Location",choices=locVec,selected=locVec[1]),
-                              sliderInput("agg","Aggregation (minutes)",min=1,max=5,step=1,value=3))     
+                              sliderInput("agg","Aggregation (minutes)",min=1,max=5,step=1,value=2))     
     )
     )