data_tidying.Rmd

---
title: "Stat-415/615 Project - Tidying Data & EDA"
author: "Naomi Carrigg, Conor Gillingham, Emily Randolph, Connor Rempe"
date: "10/31/24"
output:
  pdf_document: 
    toc: no
    toc_depth: 2
    number_sections: no
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
library(tidyverse)
```

```{r}
## cleaning/tidying the data
AirQuality_data_raw <- read_csv("Air_Quality_History.csv")

AirQuality_data <- AirQuality_data_raw %>%
  mutate(across(starts_with("PARAMETER_NAME"), 
                ~ str_replace_all(., " ", "_") %>%
                  str_to_lower() %>%
                  str_remove_all("[^a-z_]"))) %>%
  mutate(SITE_NAME = case_match(SITE_NUM, 
                                41 ~ "River_Terrace_NE", 
                                43 ~ "McMillan_NW", 
                                50 ~ "Takoma_Recreation_NW",
                                51 ~ "Anacostia_Freeway_NE", 
                                53 ~ "Greenleaf_Recreation_SW", 
                                42 ~ "Hains_Point_SW")) %>%
  select(-c(LONGITUDE, LATITUDE, STATE_CODE, STATE_NAME, COUNTY_NAME, POC, DATUM, OBJECTID, UNITS_OF_MEASURE, METHOD_CODE, METHOD_NAME, AQSID, CITY_NAME, CBSA_NAME, ADDRESS, LOCAL_SITE_NAME, SAMPLE_DURATION, EVENT_TYPE)) %>%
  mutate(DATETIME_LOCAL = as.POSIXct(DATETIME_LOCAL, tz = "UTC"),
         Year = year(DATETIME_LOCAL),
         Month = month(DATETIME_LOCAL, label = TRUE, abbr = TRUE) %>% as.character()) %>%
  mutate(Season = case_when(
    Month %in% c("Dec", "Jan", "Feb") ~ 'Winter',
    Month %in% c("Mar", "Apr", "May") ~ 'Spring',
    Month %in% c("Jun", "Jul", "Aug") ~ 'Summer',
    TRUE ~ 'Fall'
  ))

# Grouping and summarizing
AirQuality_data_means <- AirQuality_data %>%
  group_by(Season, SITE_NAME, Month, Year, PARAMETER_NAME) %>%
  summarize(ARITHMETIC_MEAN = mean(ARITHMETIC_MEAN, na.rm = TRUE), .groups = 'drop')

AirQuality_data_aqi <- AirQuality_data %>%
  group_by(Season, SITE_NAME, Month, Year, PARAMETER_NAME) %>%
  summarize(AQI = mean(AQI, na.rm = TRUE), .groups = 'drop')

# Pivot the data, 
# this is where the issue is with the NAs  because there are some cases which don't have any data for the 5  groups, we can keep these and just have a ton of NA's or choose which pollutants we want to focus on
AirQuality_data_means <- AirQuality_data_means %>%
  pivot_wider(names_from = PARAMETER_NAME, values_from = ARITHMETIC_MEAN)

AirQuality_data_aqi <- AirQuality_data_aqi %>%
  pivot_wider(names_from = PARAMETER_NAME, values_from = AQI)
```

```{r}
head(AirQuality_data_means)
tail(AirQuality_data_means)
head(AirQuality_data_aqi)
tail(AirQuality_data_aqi)
```


```{r}
#Columns with at least 1 non-NA AQI
AirQuality_data_aqi_reduced <- subset(AirQuality_data_aqi, select = c("Season", "SITE_NAME","Month", "Year", "carbon_monoxide","nitrogen_dioxide_no", "pm__local_conditions", "ozone", "pm_total_um_stp","sulfur_dioxide"))
```

```{r}
# means df with columns of interest
reduced_data <- AirQuality_data_means %>% 
  subset(select = c("Season", "SITE_NAME","Month", "Year", "barometric_pressure", "carbon_monoxide", "nitrogen_dioxide_no","outdoor_temperature", "pm__local_conditions","relative_humidity"	, "wind_direction__resultant", "wind_speed__resultant"))
```

```{r}
reduced_data %>% 
  ggplot(aes(x = barometric_pressure, y = carbon_monoxide)) +
  geom_point() +
  geom_smooth(method = 'lm', se = FALSE, color = 'red') +
  facet_wrap(~Year)
```


```{r}
# Get Descriptive Statistics for numeric colmns in means
descriptive_stats_means <- data.frame()

for (col in names(AirQuality_data_means)) {
  if (is.numeric(AirQuality_data_means[[col]])) {
    working_means <- AirQuality_data_means %>%
      summarise(
        variable = col,
        mean = mean(.data[[col]], na.rm = TRUE),
        median = median(.data[[col]], na.rm = TRUE),
        sd = sd(.data[[col]], na.rm = TRUE),
        min = min(.data[[col]], na.rm = TRUE),
        max = max(.data[[col]], na.rm = TRUE),
        n = sum(!is.na(.data[[col]]))
      )
        descriptive_stats_means <- bind_rows(descriptive_stats_means, working_means)
  }
}

print(descriptive_stats_means)
```


```{r}
# Descriptive stats for AQI
descriptive_stats_aqi <- data.frame()

for (col in names(AirQuality_data_aqi_reduced)) {
  if (is.numeric(AirQuality_data_aqi_reduced[[col]])) {
    working_means_aqi <- AirQuality_data_aqi_reduced %>%
      summarise(
        variable = col,
        mean = mean(.data[[col]], na.rm = TRUE),
        median = median(.data[[col]], na.rm = TRUE),
        sd = sd(.data[[col]], na.rm = TRUE),
        min = min(.data[[col]], na.rm = TRUE),
        max = max(.data[[col]], na.rm = TRUE),
        n = sum(!is.na(.data[[col]]))
      )
        descriptive_stats_aqi <- bind_rows(descriptive_stats_aqi, working_means_aqi)
  }
}

print(descriptive_stats_aqi)
```


```{r}
# Create a list of histograms for the specified columns
graph <- function(pollutant, name) {
  ggplot(reduced_data, aes(x = pollutant)) +
    geom_histogram(binwidth = 1, fill = 'lightblue', color = 'black', na.rm = TRUE) +
    labs(title = paste0('Histogram of ', name), x = name, y = 'Frequency') +
    theme_minimal()
}

graph(reduced_data$barometric_pressure, 'Barometric Pressure')
graph(reduced_data$carbon_monoxide, 'Carbon Monoxide')
graph(reduced_data$nitrogen_dioxide_no, 'Nitrogen Dioxide')
graph(reduced_data$outdoor_temperature, 'Temperature')
graph(reduced_data$pm__local_conditions, 'Particulate Matter')
graph(reduced_data$relative_humidity, 'Relative Humidity')
graph(reduced_data$wind_direction__resultant, 'Wind Direction')
graph(reduced_data$wind_speed__resultant, 'Wind Speed')
```

# Model Fitting
# Model 1: Predicting Carbon Monoxide with Nitrogen Dioxide and season using AQI
```{r}
model_1_data <- na.omit(subset(AirQuality_data_aqi_reduced, select = c("Season", "Month", "Year", "SITE_NAME", "nitrogen_dioxide_no", "carbon_monoxide")))
model_1 <- lm(carbon_monoxide ~ Season + nitrogen_dioxide_no, data = model_1_data)
summary(model_1)
```

# Hypothesis test (Can Nitrogen be dropped from the model)
```{r}
reduced <- lm(carbon_monoxide ~ Season, data = model_1_data)
anova(reduced, model_1)
```
Null is that the coefficient is zero, at significance level of 0.05 we reject the null and say that carbon monoxide cannot be dropped from the model. Suggests that two pollutants do occur in tandem

# Hypothesis test for Overall Significance

```{r}
null <- lm(carbon_monoxide ~ 1 , data = model_1_data)
anova(null,model_1)
```
Model is significant overall.
\newpage

# Model 2: Predicting Nitrogen Dioxide AQI using all other predictors. Trying to see if relationship exists in the opposite direction and whether location has an effect.
```{r}
model_2_data <- na.omit(subset(AirQuality_data_aqi_reduced, select = c("Season", "Month", "Year", "SITE_NAME", "nitrogen_dioxide_no")))

model_2_red <- lm(nitrogen_dioxide_no ~ SITE_NAME + Season + Month + Year, data = model_1_data)

model_2_full <- lm(nitrogen_dioxide_no ~ SITE_NAME + Season + Month + Year + carbon_monoxide, data = model_1_data)

anova(model_2_red, model_2_full)
```
\newpage
```{r}
summary(model_2_full)
summary(model_2_red)
```

Carbon monoxide can be dropped from the model when we include season, month, year, and Site_Name. Now we perform best subset selection on this model
\newpage
```{r}
step(model_2_red, direction = "both")
```
Only site name and month are retained, using Anacostia NE and April as baselines
\newpage
```{r}
model_fin <- lm(nitrogen_dioxide_no ~ SITE_NAME + Month, data = model_2_data)
summary(model_fin)
```


```{r}
plot(model_fin$resi)
```


```{r}
qqnorm(model_fin$resi)
qqline(model_fin$resi)
```


Residual plots indicate that linear model is appropriate.

```{r}
library(lmtest)
```

```{r}
dwtest(model_fin)
```
Caveat, DW test reveals autocorrelation in this mod

```{r}
#rstudent(model_fin) # studentized deleted residuals
# dffits(model_fin) # DF fits
# dfbetas(model_fin) # standardized DF betas
# cooks.distance(model_fin) # cook's dist
# hatvalues(model_fin) # leverage
```

```{r}
n <- 119 # sample size
p <- 2 # 5 parameters

par(mfrow = c(2, 2))
plot(rstudent(model_fin), ylab = 'Studentized Deleted Residual')
abline(qt(1-0.1/(2*n), df = n - p - 1), 0, lty = 2)
abline(qt(0.1/(2*n), df = n - p - 1), 0, lty = 2)

plot(dffits(model_fin), ylab = 'DF Fits')
abline(1, 0, lty = 2)
abline(1, 0, lty = 2)

plot(cooks.distance(model_fin), ylab = "Cook's Distance")
abline(qf(0.5, df1 = 5, df2 = n - p), 0, lty = 2)

plot(hatvalues(model_fin), ylab = 'Leverege')
abline(2*p/n, 0, lty = 2)

library(car)
vif(model_fin)

influenceIndexPlot(model_fin)
```

# Model 3: Does weather have an affect?

```{r}
model_3_data <- na.omit(subset(reduced_data, select = c("Season", "Month", "Year", "SITE_NAME", "nitrogen_dioxide_no", "carbon_monoxide","outdoor_temperature", "relative_humidity"	, "wind_direction__resultant", "wind_speed__resultant")))

dim(model_3_data)
```

\newpage
```{r}
model_3 <- lm(nitrogen_dioxide_no ~ ., data = model_3_data)
summary(model_3)
```

\newpage
```{r}
step(model_3, direction = "both")
```
\newpage
```{r}
model_3_step <- lm(formula = nitrogen_dioxide_no ~ Month + Year + SITE_NAME + 
    carbon_monoxide + outdoor_temperature + relative_humidity + 
    wind_direction__resultant + wind_speed__resultant, data = model_3_data)
summary(model_3_step)
```
\newpage
Anova test for significance of weather predictors
```{r}
model_3_red <- lm(formula = nitrogen_dioxide_no ~ Month + Year + SITE_NAME + 
    carbon_monoxide, data = model_3_data)

anova(model_3_red, model_3_step)
```
All weather variables cannot be dropped from the model, weather has some impact on concentration. 

```{r}
library(glmnet)
x_var <- model.matrix(model_3_step)
y_var <- as.numeric(model_3_data[[ "nitrogen_dioxide_no"]])
lambda_seq <- 10^seq(2, -1, by = -0.1)

ridge <- glmnet(x_var, y_var, alpha = 0, lambda = seq(0, 10, 0.01))
plot(ridge)

ridge_cv <- cv.glmnet(x_var, y_var, alpha = 0, lambda = seq(0, 10, 0.01))
best_lambda <- ridge_cv$lambda.min

ridge_out <- glmnet(x_var, y_var, alpha = 0, lambda = best_lambda)
summary(ridge_out)
coef(ridge_out)

predict(ridge, ridge_cv$lambda.min, newx = x_var, type = 'coefficients')

plot(ridge_cv)

y_predicted <- predict(ridge_out, s = best_lambda, newx = x_var)
sst <- sum((y_var - mean(y_var))^2)
sse <- sum((y_predicted - y_var)^2)
rsq <- 1 - sse/sst
rsq

best_lambda
```


# Overall findings
1. Concentration of other pollutants doesn't have a significant effect when Location and Month are included in the model
2. In general, winter and fall months tend to have higher AQI 
3. Season and Site location are strong predictors, suggests disparities accross DC. NE worse off than NW. 
4. It appears that weather has an affect