482 create tadamonitoringlocationidentifier in tada autoclean

.Rbuildignore

@@ -9,3 +9,4 @@ readme.html
 .gitignore
 ^articles
 ^vignettes/articles$
+

.gitignore

@@ -23,6 +23,3 @@ test_function.R
 _snaps
 
 testing_log.txt
-
-# test data from AK
-AK_EPATADA_troubleshooting

DESCRIPTION

@@ -12,7 +12,9 @@ Authors@R:
            comment = c(ORCID = "0000-0002-0615-6087")),
     person(given = "Hillary",
            family = "Marler",
-           role = "aut"),
+           role = "aut",
+           email = "[email protected]",
+           comment = c(ORCID = "0009-0006-0323-1172")),
     person(given = "Jacob",
            family = "Greif",
            role = "aut"),
@@ -72,7 +74,9 @@ Imports:
     nhdplusTools,
     arcgislayers,
     httr,
-    jsonlite
+    jsonlite,
+    units,
+    igraph
 Depends: 
     R (>= 3.5.0)
 Suggests:

R/CriteriaComparison.R

@@ -181,15 +181,17 @@ TADA_CreatePairRef <- function(.data, ph = TRUE, hardness = TRUE, temp = TRUE,
 #' Pair Results for Numeric Criteria Calculation (UNDER ACTIVE DEVELOPMENT)
 #'
 #' This function pairs TADA results with results from specified characteristics from the same
-#' MonitoringLocation within a user-specified time window to facilitate the calculation of numeric
-#' criteria. The columns created by TADA_AutoClean are required to run this function. If they are not
-#' present in the data frame, the function will stop and print an error message.
+#' TADA.MonitoringLocation within a user-specified time window to facilitate the calculation of 
+#' numeric criteria. The columns created by TADA_AutoClean are required to run this function. If 
+#' they are not present in the data frame, the function will stop and print an error message.
 #'
 #' Users can provide a pairing reference file (can be created using TADA_CreatePairRef) to specify
 #' which combinations of TADA.CharacteristicName, TADA.ResultMeasure.MeasureUnit,
 #' TADA.MethodSpeciationName, and TADA.ResultSampleFractionText should be used for hardness, pH,
-#' temperature, salinity, chloride or other user-defined groups. If no ref is specified, all possible
-#' combinations for hardness, pH, temperature, salinity and chloride will be used.
+#' temperature, salinity, chloride or other user-defined groups. If no ref is specified, all 
+#' possible combinations for hardness, pH, temperature, salinity and chloride will be used. It is
+#' highly recommended that users perform all unit conversion and synonym harmonization before using
+#' TADA_PairForCriteriaCalc.
 #'
 #' @param .data TADA dataframe
 #'
@@ -277,7 +279,7 @@ TADA_PairForCriteriaCalc <- function(.data, ref = "null", hours_range = 4) {
       ) %>%
       dplyr::select(
         TADA.CharacteristicName, TADA.ResultMeasureValue, TADA.ResultMeasure.MeasureUnitCode,
-        ActivityIdentifier, MonitoringLocationIdentifier, ActivityStartDateTime,
+        ActivityIdentifier, TADA.MonitoringLocationIdentifier, ActivityStartDateTime,
         TADA.ResultSampleFractionText, TADA.MethodSpeciationName
       ) %>%
       dplyr::left_join(ref.subset,
@@ -335,11 +337,11 @@ TADA_PairForCriteriaCalc <- function(.data, ref = "null", hours_range = 4) {
       dplyr::filter(
         !ResultIdentifier %in% pair.activityid$ResultIdentifier,
         !is.na(ActivityStartDateTime),
-        MonitoringLocationIdentifier %in% pair.subset$MonitoringLocationIdentifier
+        TADA.MonitoringLocationIdentifier %in% pair.subset$TADA.MonitoringLocationIdentifier
       ) %>%
       dplyr::left_join(pair.subset2,
         relationship = "many-to-many",
-        by = dplyr::join_by(MonitoringLocationIdentifier)
+        by = dplyr::join_by(TADA.MonitoringLocationIdentifier)
       ) %>%
       dplyr::group_by(ResultIdentifier) %>%
       # Figure out fastest time comparison method - needs to be absolute time comparison

R/Filtering.R

@@ -44,7 +44,7 @@ TADA_FieldCounts <- function(.data, display = c("key", "most", "all"), character
       "ActivityMediaSubdivisionName",
       "ActivityCommentText",
       "ResultCommentText",
-      "MonitoringLocationTypeName",
+      "TADA.MonitoringLocationTypeName",
       "StateCode",
       "OrganizationFormalName",
       "TADA.CharacteristicName",
@@ -79,8 +79,10 @@ TADA_FieldCounts <- function(.data, display = c("key", "most", "all"), character
       "ActivityRelativeDepthName",
       "ProjectIdentifier",
       "ProjectName",
+      "TADA.MonitoringLocationIdentifier",
       "MonitoringLocationIdentifier",
       "MonitoringLocationName",
+      "MonitoringLocationTypeName",
       "ActivityCommentText",
       "SampleAquifer",
       "HydrologicCondition",
@@ -112,7 +114,6 @@ TADA_FieldCounts <- function(.data, display = c("key", "most", "all"), character
       "ResultDetectionQuantitationLimitUrl",
       "DetectionQuantitationLimitTypeName",
       "ProviderName",
-      "MonitoringLocationTypeName",
       "MonitoringLocationDescriptionText",
       "HUCEightDigitCode",
       "HorizontalCollectionMethodName",
@@ -191,10 +192,14 @@ TADA_FieldValuesTable <- function(.data, field = "null", characteristicName = "n
   if (!field %in% names(.data)) {
     stop("Field input does not exist in dataset. Please populate the 'field' argument with a valid field name. Enter ?TADA_FieldValuesTable in console for more information.")
   }
+
+  # change NAs to "NA" (character string)
+  .data[[field]][is.na(.data[[field]])] <- "NA"
 
   # filter to characteristic if provided
   if (!characteristicName %in% c("null")) {
-    .data <- subset(.data, .data$TADA.CharacteristicName %in% c(characteristicName))
+    .data <- .data %>%
+      dplyr::filter(TADA.CharacteristicName %in% characteristicName)
     if (dim(.data)[1] < 1) {
       stop("Characteristic name(s) provided are not contained within the input dataset. Note that TADA converts characteristic names to ALL CAPS for easier harmonization.")
     }
@@ -282,11 +287,9 @@ TADA_AnalysisDataFilter <- function(.data,
   # import MonitoringLocationTypeNames and TADA.Media.Flags
   sw.sitetypes <- utils::read.csv(system.file("extdata", "WQXMonitoringLocationTypeNameRef.csv", package = "EPATADA")) %>%
     dplyr::select(Name, TADA.Media.Flag) %>%
-    dplyr::rename(
-      ML.Media.Flag = TADA.Media.Flag,
-      MonitoringLocationTypeName = Name
-    )
-
+    dplyr::rename(ML.Media.Flag = TADA.Media.Flag) %>%
+    dplyr::mutate(MonitoringLocationTypeName = toupper(Name)) %>%
+    dplyr::select(-Name)
 
   # add TADA.Media.Flag column
   .data <- .data %>%
@@ -304,7 +307,7 @@ TADA_AnalysisDataFilter <- function(.data,
       ActivityMediaSubdivisionName == "Surface Water" ~ "Surface Water",
       !ActivityMediaName %in% c("WATER", "Water", "water") ~ ActivityMediaName
     )) %>%
-    # add TADA.Media.Flag for additional rows based on MonitoringLocationTypeName
+    # add TADA.Media.Flag for additional rows based on TADA.MonitoringLocationTypeName
     dplyr::left_join(sw.sitetypes, by = "MonitoringLocationTypeName") %>%
     dplyr::mutate(
       TADA.Media.Flag = ifelse(is.na(TADA.Media.Flag),

R/GeospatialFunctions.R

@@ -2,10 +2,12 @@
 #'
 #' Transform a Water Quality Portal dataframe into a geospatial sf object.
 #'
-#' Adds one new column to input dataframe, 'geometry', which allows for mapping and additional geospatial capabilities. Check out the TADAModule2.Rmd for an example workflow.
+#' Adds one new column to input dataframe, 'geometry', which allows for mapping and additional 
+#' geospatial capabilities. Check out the TADAModule2.Rmd for an example workflow.
 #'
 #' @param .data A dataframe created by `TADA_DataRetrieval()`.
-#' @param crs The coordinate reference system (CRS) you would like the returned point features to be in. The default is CRS 4326 (WGS84).
+#' @param crs The coordinate reference system (CRS) you would like the returned point features to 
+#' be in. The default is CRS 4326 (WGS84).
 #'
 #' @return The original TADA Water Quality Portal dataframe but as geospatial sf point objects.
 #'
@@ -111,11 +113,16 @@ TADA_MakeSpatial <- function(.data, crs = 4326) {
 
 #' fetchATTAINS
 #'
-#' Fetches ATTAINS features (state- or tribe- or other entity- submitted points, lines, and polygons representing their assessment units; and the EPA snapshot of the associated NHDPlus HR catchments that the state- or tribe- or other entity- submitted features fall within) within a bounding box produced from a set of TADA spatial features.
+#' Fetches ATTAINS features (state- or tribe- or other entity- submitted points, lines, and polygons 
+#' representing their assessment units; and the EPA snapshot of the associated NHDPlus HR catchments 
+#' that the state- or tribe- or other entity- submitted features fall within) within a bounding box 
+#' produced from a set of TADA spatial features.
 #'
 #' @param .data A dataframe developed using `TADA_DataRetrieval()` or `TADA_MakeSpatial()`.
-#' @param catchments_only Whether to return just the summarized ATTAINS catchment features, or both the catchments and raw ATTAINS features. TRUE or FALSE.
-#' @return Spatial features (ATTAINS_catchments, ATTAINS_points, ATTAINS_lines, and ATTAINS_polygons) that are within the spatial bounding box of water quality observations.
+#' @param catchments_only Whether to return just the summarized ATTAINS catchment features, or both 
+#' the catchments and raw ATTAINS features. TRUE or FALSE.
+#' @return Spatial features (ATTAINS_catchments, ATTAINS_points, ATTAINS_lines, and 
+#' ATTAINS_polygons) that are within the spatial bounding box of water quality observations.
 #'
 #' @seealso [TADA_MakeSpatial()]
 #' @seealso [TADA_DataRetrieval()]
@@ -266,7 +273,8 @@ fetchATTAINS <- function(.data, catchments_only = FALSE) {
   }
 
   # If the area of the bbox is massive (about the area of California or larger), AND there
-  # aren't that many actual monitoring locations (100)... OR the bbox is about the size of New Hampshire, and the observations are under 25...
+  # aren't that many actual monitoring locations (100)... OR the bbox is about the size of 
+  # New Hampshire, and the observations are under 25...
   # ... speed up processing by going site-by-site:
   if (nrow(.data) <= 100 & as.numeric(sf::st_area(sf::st_as_sfc(bbox_raw))) >= 4e+11 || nrow(.data) <= 25 & as.numeric(sf::st_area(sf::st_as_sfc(bbox_raw))) >= 1e+11) {
     catchment_features <- vector("list", length = nrow(.data))
@@ -458,13 +466,20 @@ fetchATTAINS <- function(.data, catchments_only = FALSE) {
 
 #' fetchNHD
 #'
-#' Fetches NHD features from either the high resolution or medium resolution version of the National Hydrography Dataset (NHD) that intersect catchments containing TADA Water Quality Portal observations.
+#' Fetches NHD features from either the high resolution or medium resolution version of the National 
+#' Hydrography Dataset (NHD) that intersect catchments containing TADA Water Quality Portal 
+#' observations.
 #'
-#' @param .data A dataframe created by `TADA_DataRetrieval()` or the geospatial equivalent made by `TADA_MakeSpatial()`.
-#' @param resolution Whether to download the NHDPlus HiRes resolution ("Hi") or medium NHDPlus V2 resolution ("Med") version of the National Hydrography Dataset (NHD). Default is "Hi".
-#' @param features Which NHD features to return: "catchments", "flowlines", "waterbodies", or any combination.
+#' @param .data A dataframe created by `TADA_DataRetrieval()` or the geospatial equivalent made by 
+#' `TADA_MakeSpatial()`.
+#' @param resolution Whether to download the NHDPlus HiRes resolution ("Hi") or medium NHDPlus V2 
+#' resolution ("Med") version of the National Hydrography Dataset (NHD). Default is "Hi".
+#' @param features Which NHD features to return: "catchments", "flowlines", "waterbodies", or any 
+#' combination.
 #'
-#' @return A list containing all selected NHD features associated with the WQP observations of interest. Or, if a single feature type is selected, a single geospatial object instead of a list. Default is "catchments" only.
+#' @return A list containing all selected NHD features associated with the WQP observations of 
+#' interest. Or, if a single feature type is selected, a single geospatial object instead of a list. 
+#' Default is "catchments" only.
 #'
 #' @seealso [TADA_DataRetrieval()]
 #' @seealso [TADA_MakeSpatial()]
@@ -1095,8 +1110,8 @@ TADA_GetATTAINS <- function(.data, fill_catchments = FALSE, resolution = "Hi", r
     print("There are no ATTAINS catchments associated with these WQP observations. Returning an empty data frame for `TADA_with_ATTAINS`.")
 
     if (fill_catchments == FALSE) {
-      # If there are no intersecting ATTAINS catchments, fill_catchments = FALSE, and return_sf = TRUE, return empty sf features with the
-      # empty TADA_with_ATTAINS df.
+      # If there are no intersecting ATTAINS catchments, fill_catchments = FALSE, and 
+      # return_sf = TRUE, return empty sf features with the empty TADA_with_ATTAINS df.
       if (return_sf == TRUE) {
         ATTAINS_catchments <- NULL
         ATTAINS_lines <- NULL
@@ -1111,8 +1126,8 @@ TADA_GetATTAINS <- function(.data, fill_catchments = FALSE, resolution = "Hi", r
           "ATTAINS_polygons" = ATTAINS_polygons
         ))
       } else {
-        # If there are no intersecting ATTAINS catchments, fill_catchments = FALSE, and return_sf = FALSE, just return the
-        # empty TADA_with_ATTAINS df.
+        # If there are no intersecting ATTAINS catchments, fill_catchments = FALSE, and 
+        # return_sf = FALSE, just return the empty TADA_with_ATTAINS df.
         return(no_ATTAINS_data)
       }
     } else if (fill_catchments == TRUE) {

R/RequiredCols.R

@@ -171,7 +171,9 @@ require.cols <- c(
   "StateCode",
   "CountyCode",
   "MonitoringLocationName", # required
+  "TADA.MonitoringLocationName", # generated
   "MonitoringLocationTypeName",
+  "TADA.MonitoringLocationTypeName", #generated
   "MonitoringLocationDescriptionText",
   "LatitudeMeasure",
   "TADA.LatitudeMeasure", # generated
@@ -182,6 +184,9 @@ require.cols <- c(
   "HUCEightDigitCode",
   "MonitoringLocationIdentifier", # required
   "TADA.MonitoringLocationIdentifier",
+  "TADA.NearbySites.Flag", # generated,
+  "TADA.NearbySiteGroup", #generated
+
   # Groundwater fields, used for auto filtering for assessment use case but should not be required to have in TADA template
   "AquiferName", # filter, groundwater
   "AquiferTypeName", # filter