initial commit

Chicago · Jun 13, 2017 · 43ffc39 · 43ffc39
commit 43ffc39
Show file tree

Hide file tree

Showing 9 changed files with 300 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,5 @@
+.Rproj.user
+.Rhistory
+.RData
+.Ruserdata
+credentials/*
diff --git a/Functions/usePackage.R b/Functions/usePackage.R
@@ -0,0 +1,11 @@
+# The usePackage() function checks to see if the package is installed, if the 
+# the package is not installed then the package is installed. 
+#Once the package is installed, or was previously installed
+#it is put to use through the require() statement.
+
+usePackage <- function(p) 
+{
+  if (!is.element(p, installed.packages()[,1]))
+    install.packages(p, repos = "https://cloud.r-project.org", dep = TRUE)
+  require(p, character.only = TRUE)
+}
diff --git a/README.md b/README.md
@@ -0,0 +1,19 @@
+# Notes
+
+The file ```model.Rds``` is obtained by running ```Master.R``` in the 
+[E. coli Model](https://github.com/Chicago/e-coli-beach-predictions).
+The model object is created in the ```modelEcoli.R``` [file](https://github.com/Chicago/e-coli-beach-predictions/blob/master/Functions/modelEcoli.R). Add a line of code
+to save the model object as an Rds file, and configure the training set in the ```Master.R``` [file](https://github.com/Chicago/e-coli-beach-predictions/blob/master/Master.R). Make sure you set the
+[kFolds variable](https://github.com/Chicago/e-coli-beach-predictions/blob/master/Master.R#L51) to FALSE.
+
+# Socrata Credentials
+
+To authenticate with Socrata and upload predictions, create files to save login 
+and app token details in the `credentials` folder, named as follows:
+
+* `email.txt`
+* `password.txt`
+* `token.txt`
+
+```App.R``` will read these credentials and use them to make send the data
+to the Socrata dataset.
diff --git a/app.R b/app.R
@@ -0,0 +1,122 @@
+#------------------------------------------------------------------------------#
+# A script to generate Clear Water Predictions
+# This script checks the data portal to see if today's DNA test results are posted
+# If yes, then it upserts the predictions to the data portal
+# It is run every 5 minutes
+#
+# Author: Nick Lucius
+#------------------------------------------------------------------------------#
+
+rm(list=ls())
+
+#------------------------------------------------------------------------------#
+# Source Functions                                                             
+#------------------------------------------------------------------------------#
+
+sourceDir <- function(path, trace = TRUE, ...) {
+  for (nm in list.files(path, pattern = "\\.[Rr]$")) {
+    if(trace) cat(nm,":")           
+    source(file.path(path, nm), ...)
+    if(trace) cat("\n")
+  }
+}
+
+sourceDir(paste(getwd(),"/Functions",sep=""))
+
+#------------------------------------------------------------------------------#
+# Load libraries                                                             
+#------------------------------------------------------------------------------#
+
+usePackage("RSocrata")
+usePackage("randomForest")
+
+#------------------------------------------------------------------------------#
+# Set variables                                                             
+#------------------------------------------------------------------------------#
+
+today <- Sys.Date()
+modelPath <- paste0(getwd(),"/data/model.Rds")
+
+# Socrata variables
+app_token <- readLines("credentials/token.txt")
+email <- readLines("credentials/email.txt")
+password <- readLines("credentials/password.txt")
+
+# bring in lat/longs for each beach, which must be included in the upsert
+cleanBeachNames <- read.csv("csv/cleanbeachnames.csv")
+locationLookup <- cleanBeachNames[,c("Short_Names","Latitude","Longitude")]
+locationLookup <- unique(locationLookup)
+for (row in c(1:length(locationLookup$Short_Names))) {
+  locationLookup$Location[row] <- paste0("(", locationLookup$Latitude[row], ", ", locationLookup$Longitude[row], ")")
+}
+displayNames <- read.csv("csv/beach-display-names.csv")
+
+beaches <- c("12th","31st","39th","57th","Albion","Foster","Howard","Jarvis","Juneway","Leone",       
+             "North Avenue","Oak Street","Ohio","Osterman","Rogers") # beaches being predicted
+allBeaches <- c("12th","31st","39th","57th","63rd","Albion","Calumet","Foster",
+                "Howard","Jarvis","Juneway","Leone","Montrose","North Avenue","Oak Street",
+                "Ohio","Osterman","Rainbow","Rogers","South Shore") # all beaches
+beaches <- factor(beaches, levels = allBeaches)
+
+#------------------------------------------------------------------------------#
+# Download data needed for the model                                                             
+#------------------------------------------------------------------------------#
+
+#Pull latest DNA tests from Data Portal and determine if we have the 5 beaches that the model needs
+labPortal <- read.socrata("https://data.cityofchicago.org/Parks-Recreation/Beach-Lab-Data/2ivx-z93u",
+                          app_token = "ew2rEMuESuzWPqMkyPfOSGJgE")
+dates <- labPortal$DNA.Sample.Timestamp 
+dates <- strftime(dates, format = "%Y-%m-%d")
+todaysLabs <- labPortal[dates == today & !is.na(dates),]
+readyToModel <- "Calumet" %in% todaysLabs$Beach & 
+  "63rd Street" %in% todaysLabs$Beach & 
+  "Rainbow" %in% todaysLabs$Beach & 
+  "Montrose" %in% todaysLabs$Beach & 
+  "South Shore" %in% todaysLabs$Beach
+
+#------------------------------------------------------------------------------#
+# Generate Predictions                                                             
+#------------------------------------------------------------------------------
+
+# if we have all the inputs, run the model and send predictions to Data Portal
+if (readyToModel) {
+
+  input <- data.frame("Client.ID" = beaches,
+                      "Date" = today,
+                      "n63rd_DNA.Geo.Mean" = todaysLabs[todaysLabs$Beach == "63rd Street","DNA.Reading.Mean"],
+                      "South_Shore_DNA.Geo.Mean" = todaysLabs[todaysLabs$Beach == "South Shore","DNA.Reading.Mean"],
+                      "Montrose_DNA.Geo.Mean" = todaysLabs[todaysLabs$Beach == "Montrose","DNA.Reading.Mean"],
+                      "Calumet_DNA.Geo.Mean" = todaysLabs[todaysLabs$Beach == "Calumet","DNA.Reading.Mean"],
+                      "Rainbow_DNA.Geo.Mean" = todaysLabs[todaysLabs$Beach == "Rainbow","DNA.Reading.Mean"])
+  model <- readRDS(modelPath)
+  predictions <- cbind(input,"prediction" = predict(model,input))
+
+  # format output
+  output <- predictions[,c(1,2,8)]
+  names(output)[1:3] <- c("beach_name", "date", "predicted_level")
+  output$predicted_level <- round(output$predicted_level, digits = 1)
+  output$prediction_source <- "DNA Model"
+  output <- output[,c(1,2,4,3)]
+  output <- merge(output, locationLookup, by.x = "beach_name", by.y = "Short_Names")
+
+  # change beach_name to match current names
+  output <- merge(output, displayNames, by = "beach_name")
+  output$beach_name <- output$display_name
+  output$display_name <- NULL
+
+  # add record ID
+  output$recordid <- paste0(gsub(" ", "", output$beach_name, fixed = TRUE), strftime(today, format = "%Y%m%d"))
+  output$recordid <- gsub("\\(", "", output$recordid)
+  output$recordid <- gsub("\\)", "", output$recordid)
+
+  result <- write.socrata(dataframe = output, 
+                          dataset_json_endpoint = "https://data.cityofchicago.org/resource/xvsz-3xcj.json", 
+                          update_mode = "UPSERT",
+                          email = email,
+                          password = password,
+                          app_token = app_token)
+  print(paste0("Predictions for ", today, " sent to Data Portal"))
+  print(result)
+} else {
+  print(paste0("Not enough DNA test results to issue a prediction at ", Sys.time()))
+}
diff --git a/clear-water-app.Rproj b/clear-water-app.Rproj
@@ -0,0 +1,13 @@
+Version: 1.0
+
+RestoreWorkspace: Default
+SaveWorkspace: Default
+AlwaysSaveHistory: Default
+
+EnableCodeIndexing: Yes
+UseSpacesForTab: Yes
+NumSpacesForTab: 2
+Encoding: UTF-8
+
+RnwWeave: Sweave
+LaTeX: pdfLaTeX
diff --git a/csv/beach-display-names.csv b/csv/beach-display-names.csv
@@ -0,0 +1,16 @@
+beach_name,display_name
+12th,12th Street
+31st,Margaret T Burroughs (31st)
+39th,Oakwood
+57th,57th Street
+Albion,Hartigan (Albion)
+Foster,Foster
+Howard,Howard
+Jarvis,Marion Mahony Griffin (Jarvis)
+Juneway,Juneway
+Leone,Leone
+North Avenue,North Avenue
+Oak Street,Oak Street
+Ohio,Ohio Street
+Osterman,Osterman
+Rogers,Rogers
diff --git a/csv/cleanbeachnames.csv b/csv/cleanbeachnames.csv
@@ -0,0 +1,110 @@
+Old,New,Observations Lost,Short_Names,Client.ID,Latitude,Longitude,Group,North,USGS
+12th ,12th Street,NA,12th,12,41.8638,-87.6082,4,0,7
+"12th St 1, 2",12th Street,NA,12th,12,41.8638,-87.6082,4,0,7
+12th Street,12th Street,NA,12th,12,41.8638,-87.6082,4,0,7
+12th-Street-Beach,12th Street,NA,12th,12,41.8638,-87.6082,4,0,7
+31st,Margaret T Burroughs,NA,31st,13,41.8393,-87.6072,4,0,1
+31st(Burroughs),Margaret T Burroughs,NA,31st,13,41.8393,-87.6072,4,0,1
+31st-Street-Beach,Margaret T Burroughs,NA,31st,13,41.8393,-87.6072,4,0,1
+39th,Oakwood,NA,39th,14,41.8211,-87.5976,5,0,1
+39th north,NA,2,NA,NA,NA,NA,NA,NA,NA
+39th south,NA,2,NA,NA,NA,NA,NA,NA,NA
+49th,NA,31,NA,NA,NA,NA,NA,NA,NA
+57th,57th Street,NA,57th,15,41.7911,-87.5797,5,0,1
+57th ,57th Street,NA,57th,15,41.7911,-87.5797,5,0,1
+57th Street,57th Street,NA,57th,15,41.7911,-87.5797,5,0,1
+57th-Street-Beach,57th Street,NA,57th,15,41.7911,-87.5797,5,0,1
+63rdStreet,63rd Street,NA,63rd,16,41.7827,-87.5748,5,0,1
+63rd Street,63rd Street,NA,63rd,16,41.7827,-87.5748,5,0,1
+63rd-Street-Beach,63rd Street,NA,63rd,16,41.7827,-87.5748,5,0,1
+63rd Street Beach,63rd Street,NA,63rd,16,41.7827,-87.5748,5,0,1
+Albion,Hartigan,NA,Albion,5,42.0027,-87.6564,2,1,4
+Calumet,Calumet,NA,Calumet,19,41.7142,-87.5299,6,0,2
+Calumet ,Calumet,NA,Calumet,19,41.7142,-87.5299,6,0,2
+Calumet inside barrier,NA,1,NA,NA,NA,NA,NA,NA,NA
+Calumet Inside barrier,NA,29,NA,NA,NA,NA,NA,NA,NA
+Calumet-Beach,Calumet,NA,Calumet,19,41.7142,-87.5299,6,0,2
+Calumet Beach,Calumet,NA,Calumet,19,41.7142,-87.5299,6,0,2
+Clark,NA,12,NA,NA,NA,NA,NA,NA,NA
+Clark Park,NA,9,NA,NA,NA,NA,NA,NA,NA
+Columbia-Beach,Columbia,NA,Albion,5,42.0027,-87.6564,2,1,4
+Columbia,Columbia,NA,Albion,5,42.0027,-87.6564,2,1,4
+Fargo,Fargo,NA,Jarvis,3,42.0161,-87.6647,2,1,4
+Fargo-Beach,Fargo,NA,Jarvis,3,42.0161,-87.6647,2,1,4
+Foster,Foster,NA,Foster,7,41.9785,-87.6515,1,1,3
+Foster-Beach,Foster,NA,Foster,7,41.9785,-87.6515,1,1,3
+Front 1 & 2*,NA,1,NA,NA,NA,NA,NA,NA,NA
+Hartigan,Hartigan,NA,Albion,5,42.0027,-87.6564,2,1,4
+Hartigan-Beach,Hartigan,NA,Albion,5,42.0027,-87.6564,2,1,4
+Hollywood,Osterman,NA,Osterman,6,41.9877,-87.6545,1,1,8
+Hollywood/Osterman,Osterman,NA,Osterman,6,41.9877,-87.6545,1,1,8
+Hollywood/Ostermann,Osterman,NA,Osterman,6,41.9877,-87.6545,1,1,8
+Hollywood/Thorndale*,Osterman,NA,Osterman,6,41.9877,-87.6545,1,1,8
+Howard,Howard,NA,Howard,2,42.0188,-87.6663,2,1,4
+Howard-Beach,Howard,NA,Howard,2,42.0188,-87.6663,2,1,4
+Humbodt,NA,1,NA,NA,NA,NA,NA,NA,NA
+Humboldt,NA,80,NA,NA,NA,NA,NA,NA,NA
+Humbolt,NA,222,NA,NA,NA,NA,NA,NA,NA
+Humboltd,NA,2,NA,NA,NA,NA,NA,NA,NA
+Hyde Park,57th Street,NA,57th,15,41.7911,-87.5797,5,0,1
+Jackson/63rd,63rd Street,NA,63rd,16,41.7827,-87.5748,5,0,1
+Jackson/63rd (2nd),NA,1,NA,NA,NA,NA,NA,NA,NA
+Jackson/63rd/2nd,NA,1,NA,NA,NA,NA,NA,NA,NA
+Jackson/63rd/2nd ,NA,1,NA,NA,NA,NA,NA,NA,NA
+Jarvis,Marion Mahoney Griffin,NA,Jarvis,3,42.0161,-87.6647,2,1,4
+Jarvis/ Fargo,Marion Mahoney Griffin,NA,Jarvis,3,42.0161,-87.6647,2,1,4
+Jarvis/Fargo,Marion Mahoney Griffin,NA,Jarvis,3,42.0161,-87.6647,2,1,4
+Juneway,Juneway,NA,Juneway,0,42.0224,-87.6674,2,1,4
+Juneway-Beach,Juneway,NA,Juneway,0,42.0224,-87.6674,2,1,4
+Lane,Lane,NA,Osterman,6,41.9877,-87.6545,1,1,8
+Lane-Beach,Lane,NA,Osterman,6,41.9877,-87.6545,1,1,8
+Leon,Leone,NA,Leone,4,42.0131,-87.6635,1,1,4
+Leon ,Leone,NA,Leone,4,42.0131,-87.6635,1,1,4
+Leon/Loyola,Leone,NA,Leone,4,42.0131,-87.6635,1,1,4
+Leone,Leone,NA,Leone,4,42.0131,-87.6635,1,1,4
+Leone/Loyola,Leone,NA,Leone,4,42.0131,-87.6635,1,1,4
+Leone-Beach,Leone,NA,Leone,4,42.0131,-87.6635,1,1,4
+Loyola,Loyola,NA,Leone,4,42.0131,-87.6635,1,1,4
+Loyola1,NA,1,NA,NA,NA,NA,NA,NA,NA
+Loyola2,NA,1,NA,NA,NA,NA,NA,NA,NA
+Loyola-Beach,Loyola,NA,Leone,4,42.0131,-87.6635,1,1,4
+Margaret T Burroughs,Margaret T Burroughs,NA,31st,13,41.8393,-87.6072,4,0,1
+Marion Mahoney Griffin,Marion Mahoney Griffin,NA,Jarvis,3,42.0161,-87.6647,2,1,4
+Marion-Mahoney-Griffin-Beach,Marion Mahoney Griffin,NA,Jarvis,3,42.0161,-87.6647,2,1,4
+Montrose,Montrose,NA,Montrose,8,41.9655,-87.6385,1,1,5
+Montrose Dog,NA,366,NA,NA,NA,NA,NA,NA,NA
+Montrose-Beach,Montrose,NA,Montrose,8,41.9655,-87.6385,1,1,5
+Montrose Beach,Montrose,NA,Montrose,8,41.9655,-87.6385,1,1,5
+North Ave,North Avenue,NA,North Avenue,9,41.9148,-87.6273,3,1,4
+North Avenue,North Avenue,NA,North Avenue,9,41.9148,-87.6273,3,1,4
+North Shore,North Shore,NA,Albion,5,42.0027,-87.6564,2,1,4
+North-Avenue-Beach,North Avenue,NA,North Avenue,9,41.9148,-87.6273,3,1,4
+North-Shore-Beach,North Shore,NA,Albion,5,42.0027,-87.6564,2,1,4
+Oak,Oak Street,NA,Oak Street,10,41.9032,-87.6235,3,1,6
+Oak ,Oak Street,NA,Oak Street,10,41.9032,-87.6235,3,1,6
+Oak Street,Oak Street,NA,Oak Street,10,41.9032,-87.6235,3,1,6
+Oak-Street-Beach,Oak Street,NA,Oak Street,10,41.9032,-87.6235,3,1,6
+OakStreet,Oak Street,,Oak Street,10,41.9032,-87.6235,3,1,6
+Oakwood,Oakwood,NA,39th,14,41.8211,-87.5976,5,0,1
+Oakwood-Beach,Oakwood,NA,39th,14,41.8211,-87.5976,5,0,1
+Ohio,Ohio Street,NA,Ohio,11,41.8935,-87.6152,3,1,7
+OhioStreet,Ohio Street,NA,Ohio,11,41.8935,-87.6152,3,1,7
+Ohio Street,Ohio Street,NA,Ohio,11,41.8935,-87.6152,3,1,7
+Ohio-Street-Beach,Ohio Street,NA,Ohio,11,41.8935,-87.6152,3,1,7
+Ohio Street Beach,Ohio Street,NA,Ohio,11,41.8935,-87.6152,3,1,7
+Osterman,Osterman,NA,Osterman,6,41.9877,-87.6545,1,1,8
+Osterman-Beach,Osterman,NA,Osterman,6,41.9877,-87.6545,1,1,8
+Osterman Beach,Osterman,NA,Osterman,6,41.9877,-87.6545,1,1,8
+Ping Tom Pak,NA,1,NA,NA,NA,NA,NA,NA,NA
+Ping Tom Park,NA,17,NA,NA,NA,NA,NA,NA,NA
+Pratt,Tobey Prinz,NA,Albion,5,42.0027,-87.6564,2,1,4
+Rainbow,Rainbow,NA,Rainbow,18,41.758,-87.551,5,0,9
+Rainbow-Beach,Rainbow,NA,Rainbow,18,41.758,-87.551,5,0,9
+Rainbow Beach,Rainbow,NA,Rainbow,18,41.758,-87.551,5,0,9
+Rogers,Rogers,NA,Rogers,18,42.0213,-87.6666,5,0,9
+Rogers-Beach,Rogers,NA,Rogers,18,42.0213,-87.6666,5,0,9
+South Shore,South Shore,NA,South Shore,17,41.7689,-87.5636,5,0,1
+South-Shore-Beach,South Shore,NA,South Shore,17,41.7689,-87.5636,5,0,1
+Thorndale,Lane,NA,Osterman,6,41.9877,-87.6545,1,1,8
+Tobey Prinz,Tobey Prinz,NA,Albion,5,42.0027,-87.6564,2,1,4
+Tobey-Prinz-Beach,Tobey Prinz,NA,Albion,5,42.0027,-87.6564,2,1,4
diff --git a/data/model.Rds b/data/model.Rds
diff --git a/runapp.sh b/runapp.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+Rscript --vanilla app.R
+