README.Rmd

---
output: github_document
---

<!-- README.md is generated from README.Rmd. Please edit that file -->

```{r, include = FALSE}
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)
```

# corenet

<!-- badges: start -->
[![R-CMD-check](https://github.com/nptscot/corenet/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/nptscot/corenet/actions/workflows/R-CMD-check.yaml)
<!-- badges: end -->

```{r, eval=FALSE, echo=FALSE}
# Code here used to set-up the package, saved for reference... 
# Check the pkg name is available
remotes::install_cran("available")
available::available("corenet")
# Create the package
usethis::use_description()
# Add package dependencies
usethis::use_package("sf")
usethis::use_package("tibble", type = "Suggests")
usethis::use_build_ignore("README.Rmd")
usethis::use_build_ignore("*.zip")
usethis::use_build_ignore("*.gpkg")
usethis::use_git_ignore("Data")
usethis::use_git_ignore("Doc")
# Add license via usethis (MIT):
usethis::use_mit_license("Zhao Wang")

# Add continuous integration
devtools::check()
usethis::use_github_action()
# Set-up website
usethis::use_pkgdown()
# Use GitHub pages:
usethis::use_github_pages()
# Action to build and push to the website with every commit:
usethis::use_pkgdown_github_pages()
# Create a function:
usethis::use_r("corenet")
# Create example osm_edinburgh_demo data object:
usethis::use_data_raw("osm_edinburgh_demo")

devtools::build_readme()
# Check the package again:
devtools::check()

# # Publish to CRAN (when ready):
# devtools::release()
```

`corenet` aims to provide a suite of functions designed to generate coherent networks for cycling, primarily for Scottish, but it can also be adapted for other regions or purposes, such as walking. 
`corenet` utilizes NPT data (www.npt.scot) as a key influencing parameter to facilitate the creation of these networks. Open road network (https://www.ordnancesurvey.co.uk/products/os-open-roads) serves as the base layer for the coherent network, chosen for its simplicity.

Install it with:

```{r, eval=FALSE}
if (!require("remotes")) {
  install.packages("remotes")
}
remotes::install_github("nptscot/corenet")
```

Load the package with the following for local development:

```{r, include=FALSE}
devtools::load_all()
```

## Minimal example

The package comes with example data for testing functions.
You can test the functions as follows:
```{r}
library(sf)
library(dplyr)
target_zone = zonebuilder::zb_zone("Edinburgh", n_circles = 3) |> sf::st_transform(crs = "EPSG:27700")
sf::st_crs(NPT_demo_6km) = 27700
sf::st_crs(os_edinburgh_demo_6km) = 27700
```

coherent network generation for Edinburgh city (using old input data: slow)
   user  system elapsed 
 894.13   14.56  908.35

```{r}
OS_NPT_demo = cohesive_network_prep(base_network = os_edinburgh_demo_6km, 
                                     influence_network = NPT_demo_6km, 
                                     target_zone, 
                                     crs = "EPSG:27700", 
                                     key_attribute = "road_function", 
                                     attribute_values = c("A Road", "B Road", "Minor Road"), use_stplanr = FALSE)
# mapview::mapview(OS_NPT_demo, color = "blue") + mapview::mapview(NPT_demo_6km, color = "red")
```
```{r}
#rename NPT_MM_OSM$geom as NPT_MM_OSM$geometry
OS_NPT_demo$geometry = OS_NPT_demo$geom
system.time({
CN_network_old = corenet(NPT_demo_6km, OS_NPT_demo, target_zone,
                      key_attribute = "all_fastest_bicycle_go_dutch", 
                      crs = "EPSG:27700", npt_threshold = 1500, maxDistPts = 1500,
                      road_scores = list("A Road" = 1, "B Road" = 2, "Minor Road" = 100), penalty_value = 1)    
})
mapview::mapview(CN_network, color = "blue")
CN_network_old = CN_network
```

improved coherent network generation for Edinburgh city 
   user  system elapsed 
 162.02    5.14  166.98

```{r}
os_edinburgh_demo_6km_new = sf::st_read("https://github.com/nptscot/corenet/releases/download/testing_data/os_edinburgh_demo_6km_new.geojson")

OS_NPT_demo = cohesive_network_prep(base_network = os_edinburgh_demo_6km_new, 
                                     influence_network = NPT_demo_6km, 
                                     target_zone, 
                                     crs = "EPSG:27700", 
                                     key_attribute = "road_function", 
                                     attribute_values = c("A Road", "B Road", "Minor Road"), use_stplanr = FALSE)

system.time({
CN_network_new = corenet(NPT_demo_6km, OS_NPT_demo, target_zone,
                      key_attribute = "all_fastest_bicycle_go_dutch", 
                      crs = "EPSG:27700", npt_threshold = 1500, maxDistPts = 1500,
                      road_scores = list("A Road" = 1, "B Road" = 2, "Minor Road" = 100), penalty_value = 1, group_column = "name_1")    
})
mapview::mapview(CN_network_old, color = "blue")  + mapview::mapview(CN_network_new, color = "red")                                    
```

```{r}
# The prepare_network function is invoked within the corenet function; this line allows us to examine it separately.
prepared_network = prepare_network(OS_NPT_demo, 
                                    key_attribute = "all_fastest_bicycle_go_dutch", 
                                    road_scores = list("A Road" = 1, "B Road" = 2, "Minor Road" = 100), 
                                    transform_crs = 27700)

# check weight in prepared_network
library(sfnetworks)
library(dplyr)
library(tidyr)

# Activate edges and use dplyr to summarize
summary_stats = prepared_network |>
  sfnetworks::activate("edges") |>
  as_tibble() |>  # Convert activated edges to a tibble if needed
  summarise(
    min_weight = min(weight, na.rm = TRUE),
    max_weight = max(weight, na.rm = TRUE),
    median_weight = median(weight, na.rm = TRUE),
    mean_weight = mean(weight, na.rm = TRUE)
  )

# Print the summary statistics
print(summary_stats)

# # Activate edges and select relevant columns to view
route_weights = prepared_network |>
  sfnetworks::activate("edges") |>
  as_tibble() |>  # Convert to a tibble for easier handling
  select(from, to, road_function, value, weight,penalty)  # Adjust the columns as needed

# View the first few rows in the console
print(head(route_weights))

# Or to view a specific number of routes sorted by weight:
route_weights_sorted = route_weights |>
  arrange(desc(weight))  # Use `arrange(weight)` for ascending order
print(head(route_weights_sorted, 20))  # Adjust the number to view more or fewer routes
```

```{r}
# Load necessary libraries
devtools::load_all()
library(mapview)
library(RColorBrewer)
library(purrr) # For the map and reduce functions

# Define common parameters
network_params = list(
  NPT_demo_6km = NPT_demo_6km,
  OS_NPT_demo = OS_NPT_demo,
  target_zone = target_zone,
  key_attribute = "all_fastest_bicycle_go_dutch",
  crs = "EPSG:27700",
  maxDistPts = 1500,
  road_scores = list("A Road" = 1, "B Road" = 2, "Minor Road" = 100)
)

# Define the varying npt_threshold values
# create a vector of npt_threshold values by 200 intervals from 5000 to 4000
thresholds = seq(5000, 1400, by = -200)

# Generate the networks using varying npt_threshold
CN_networks = lapply(thresholds, function(threshold) {
  corenet(NPT_demo_6km,
          OS_NPT_demo,
          target_zone = network_params$target_zone,
          key_attribute = network_params$key_attribute,
          crs = network_params$crs,
          npt_threshold = threshold,
          maxDistPts = network_params$maxDistPts,
          road_scores = network_params$road_scores)
})

# Determine the number of networks for color assignment
num_networks = length(CN_networks)

# Generate a color palette
if (num_networks > 12) {  # RColorBrewer has a max of 12 colors for most palettes
    network_colors = colorRampPalette(brewer.pal(12, "Set3"))(num_networks)
} else {
    network_colors = brewer.pal(num_networks, "Set3")
}

# Create mapview objects for each network with assigned colors
map_list <- lapply(seq_along(CN_networks), function(i) {
    # Generate a unique name for each network based on its index
    layer_name <- paste("Network", i)
    # Create the mapview object with the specified color and name
    mapview::mapview(CN_networks[[i]], color = network_colors[i], layer.name = layer_name)
})

# Combine all mapview objects into a single map object using a loop
combined_map <- mapview()  # Start with an empty mapview object
for (i in seq_along(map_list)) {
    combined_map <- combined_map + map_list[[i]]
}

# Display the combined map
combined_map
```

Group the coherent network

```{r}
grouped_network = coherent_network_group(CN_network, key_attribute = "all_fastest_bicycle_go_dutch")

mapview::mapview(grouped_network, zcol = "group", color = viridis::viridis(12), legend = TRUE)
```

Convert the coherent network to PMtiles
```{r}
netwrok = sf::st_read("cohesive networks scotland.geojson")
create_coherent_network_PMtiles(folder_path = "", city_filename = "test", cohesive_network = grouped_network)
```

# Replacing the Examples
```{r}
library(dplyr)
devtools::load_all()

# load the open roads scotland data
file_path = "inputdata/open_roads_scotland.gpkg"
open_roads_scotland = sf::read_sf(file_path)
sf::st_geometry(open_roads_scotland) = "geometry"

# load the combined network data
cnet_path = "inputdata/combined_network_tile.geojson"
combined_net = sf::read_sf(cnet_path) |>
  sf::st_transform(crs = "EPSG:27700")

# load the la_regions_2023 data
region = "Edinburgh and Lothians"
city = "City of Edinburgh"
lads = sf::read_sf("inputdata/la_regions_2023.geojson")

region_boundary = dplyr::filter(lads, Region == region & LAD23NM == city) |>
  sf::st_transform(crs = "EPSG:27700")

# Generate CN for the region
combined_net_region_boundary = combined_net[sf::st_union(region_boundary), , op = sf::st_intersects]

min_percentile_value = stats::quantile(combined_net_region_boundary$all_fastest_bicycle_go_dutch, probs = 0.94, na.rm = TRUE)

open_roads_scotland_region_boundary = open_roads_scotland[sf::st_union(region_boundary), , op = sf::st_intersects]


OS_combined_net_region_boundary = cohesive_network_prep(
  base_network = open_roads_scotland_region_boundary,
  influence_network = combined_net_region_boundary,
  region_boundary,
  crs = "EPSG:27700",
  key_attribute = "road_function",
  attribute_values = c("A Road", "B Road", "Minor Road")
)

cohesive_network_region_boundary = corenet(combined_net_region_boundary, OS_combined_net_region_boundary, region_boundary,
  key_attribute = "all_fastest_bicycle_go_dutch",
  crs = "EPSG:27700", maxDistPts = 1500, minDistPts = 2, npt_threshold = min_percentile_value,
  road_scores = list("A Road" = 1, "B Road" = 1, "Minor Road" = 100), n_removeDangles = 6, penalty_value = 1
)
mapview::mapview(cohesive_network_region_boundary)  
```

```{r}
devtools::load_all()
city = "West Lothian"
region_boundary = dplyr::filter(lads, Region == region & LAD23NM == city) |>
  sf::st_transform(crs = "EPSG:27700")

# Generate CN for the region
combined_net_region_boundary = combined_net[sf::st_union(region_boundary), , op = sf::st_intersects]

min_percentile_value = stats::quantile(combined_net_region_boundary$all_fastest_bicycle_go_dutch, probs = 0.94, na.rm = TRUE)

open_roads_scotland_region_boundary = open_roads_scotland[sf::st_union(region_boundary), , op = sf::st_intersects]


OS_combined_net_region_boundary = cohesive_network_prep(
  base_network = open_roads_scotland_region_boundary,
  influence_network = combined_net_region_boundary,
  region_boundary,
  crs = "EPSG:27700",
  key_attribute = "road_function",
  attribute_values = c("A Road", "B Road", "Minor Road")
)

cohesive_network_region_boundary = corenet(combined_net_region_boundary, OS_combined_net_region_boundary, region_boundary,
  key_attribute = "all_fastest_bicycle_go_dutch",
  crs = "EPSG:27700", maxDistPts = 1500, minDistPts = 2, npt_threshold = min_percentile_value,
  road_scores = list("A Road" = 1, "B Road" = 1, "Minor Road" = 100), n_removeDangles = 6, penalty_value = 1
)
mapview::mapview(cohesive_network_region_boundary)  
```

```{r}
region_boundary = dplyr::filter(lads, Region == region) |>
  sf::st_transform(crs = "EPSG:27700")

# Generate CN for the region
combined_net_region_boundary = combined_net[sf::st_union(region_boundary), , op = sf::st_intersects]

min_percentile_value = stats::quantile(combined_net_region_boundary$all_fastest_bicycle_go_dutch, probs = 0.9, na.rm = TRUE)

open_roads_scotland_region_boundary = open_roads_scotland[sf::st_union(region_boundary), , op = sf::st_intersects]


OS_combined_net_region_boundary = cohesive_network_prep(
  base_network = open_roads_scotland_region_boundary,
  influence_network = combined_net_region_boundary,
  region_boundary,
  crs = "EPSG:27700",
  key_attribute = "road_function",
  attribute_values = c("A Road", "B Road")
)

cohesive_network_region_boundary = corenet(combined_net_region_boundary, OS_combined_net_region_boundary, region_boundary,
  key_attribute = "all_fastest_bicycle_go_dutch",
  crs = "EPSG:27700", maxDistPts = 7000, minDistPts = 1500, npt_threshold = min_percentile_value,
  road_scores = list("A Road" = 1, "B Road" = 1), n_removeDangles = 6, penalty_value = 10000
)
mapview::mapview(cohesive_network_region_boundary)  
# load CN
CN1 = sf::read_sf("inputdata/coherent_networks/city_of_edinburgh_2024-06-03_coherent_network.geojson")
CN2 = sf::read_sf("inputdata/coherent_networks/east_lothian_2024-06-03_coherent_network.geojson")
CN3 = sf::read_sf("inputdata/coherent_networks/midlothian_2024-06-03_coherent_network.geojson")
CN4 = sf::read_sf("inputdata/coherent_networks/west_lothian_2024-06-03_coherent_network.geojson")

mapview::mapview(cohesive_network_region_boundary)   + mapview::mapview(CN1, color = "red") + mapview::mapview(CN2, color = "blue") + mapview::mapview(CN3, color = "green") + mapview::mapview(CN4, color = "yellow")

```

```{r}
cn = cycle_network(area = target_zone, NPT_network = NPT_demo_6km, length_threshold = 1)
mapview::mapview(cn)
```
```{r}
library(osmactive)
library(sf) 
library(dplyr) 
library(ggplot2)
library(gridExtra)
library(sfnetworks)
cycle_network = function(area, NPT_network, length_threshold = 1) {

  osm = osmactive::get_travel_network("Scotland", boundary = area, boundary_type = "clipsrc")
  cycle_net = osmactive::get_cycling_network(osm)
  drive_net = osmactive::get_driving_network_major(osm)
  cycle_net = osmactive::distance_to_road(cycle_net, drive_net)
  cycle_net = osmactive::classify_cycle_infrastructure(cycle_net)
  # filter cycle_net based on column bicycle is yes dismount adn designated
  cycle_net = cycle_net |>
    dplyr::filter(bicycle %in% c("yes", "dismount", "designated")) |>
    dplyr::filter(cycle_segregation == "Separated cycle track") |>
    mutate(length = as.numeric(st_length(geometry))) |>
    dplyr::filter(length > length_threshold) |>
    sf::st_transform(crs = 27700)

  snapped_lines = sf::st_snap(cycle_net, cycle_net, tolerance = 15)
  group_ids = sapply(st_geometry(snapped_lines), function(geometry, index, lines) {
    possible_near = sf::st_intersects(geometry, lines, sparse = FALSE)
    connected = which(possible_near)
    unioned = sf::st_union(st_geometry(lines[connected, ]))
    return(unioned)
  }, index = lines_index, lines = snapped_lines)

    # Create a new sf object with merged lines
    merged_lines = sf::st_sf(geometry = do.call(st_sfc, group_ids))
    merged_lines = merged_lines[!duplicated(st_as_text(merged_lines$geometry)), ]    

    network = merged_lines
    network_multilines = network[sf::st_geometry_type(network) == "MULTILINESTRING", ]
    network_lines = sf::st_cast(network_multilines, "LINESTRING")

    network = network_lines
    if (!inherits(network, "sfnetwork")) {
        network_sfn = sfnetworks::as_sfnetwork(network, directed = FALSE)
    } else {
        network_sfn = network
    }

    network_igraph = tidygraph::as_tbl_graph(network_sfn)   

    components = igraph::components(network_igraph)
    component_ids = order(components$csize, decreasing = TRUE)[1:15]  # top 10 components by size

    top_components_sfn = list()  # Initialize list to store sfnetwork components

    # Extract each of the top 10 components and convert them
    for (component_id in component_ids) {
        component_nodes = which(components$membership == component_id)
        component_subgraph = igraph::subgraph(network_igraph, component_nodes)
        if (length(component_nodes) > 0) {  # Check if there are nodes in the component
            component_sfn = sfnetworks::as_sfnetwork(component_subgraph, directed = FALSE)
            top_components_sfn[[component_id]] = component_sfn
        } else {
            top_components_sfn[[component_id]] = NULL
        }
    }

    valid_components = sapply(top_components_sfn, function(x) !is.null(x) && inherits(x, "sfnetwork"))
    all_edges = NULL

    # Loop through each component, activate edges, convert to sf, and combine
    for (i in seq_along(top_components_sfn)) {
        if (!is.null(top_components_sfn[[i]]) && inherits(top_components_sfn[[i]], "sfnetwork")) {
            # Activate edges and convert to sf
            edges_sf = top_components_sfn[[i]] |>
                sfnetworks::activate("edges") |>
                sf::st_as_sf() |>
                dplyr::mutate(component = as.factor(i))  # Add component ID
            
            # Combine into one sf object
            if (is.null(all_edges)) {
                all_edges = edges_sf
            } else {
                all_edges = rbind(all_edges, edges_sf)
            }
        }
    }

    sf::st_crs(all_edges) = 27700

    funs = list()

    name_list = names(NPT_zones) 
    for (name in name_list) {
    if (name == "geometry") {
        next  # Correctly skip the current iteration if the name is "geometry"
    } else if (name %in% c("gradient", "quietness")) {
        funs[[name]] = mean  # Assign mean function for specified fields
    } else {
        funs[[name]] = mean  # Assign sum function for all other fields
    }
    }

    filtered_OS_zones = all_edges |> 
                        sf::st_transform(27700) |> 
                        sf::st_zm()
    cycle_net_NPT =  stplanr::rnet_merge(filtered_OS_zones, NPT_zones, dist = 10, funs = funs, max_angle_diff = 10)

    summarized_data = cycle_net_NPT |>
        dplyr::group_by(component) |>
        dplyr::summarize(total_all_fastest_bicycle_go_dutch = sum(all_fastest_bicycle_go_dutch, na.rm = TRUE))

    min_percentile_value = stats::quantile(summarized_data$total_all_fastest_bicycle_go_dutch, probs = 0.3, na.rm = TRUE)
    
    summarized_data = summarized_data |> dplyr::filter(total_all_fastest_bicycle_go_dutch > min_percentile_value)

    return(summarized_data)
}
```

```{r}
cycle_network = function(area, NPT_network, length_threshold = 1) {

  osm = osmactive::get_travel_network("Scotland", boundary = area, boundary_type = "clipsrc")
  cycle_net = osmactive::get_cycling_network(osm)
  drive_net = osmactive::get_driving_network_major(osm)
  cycle_net = osmactive::distance_to_road(cycle_net, drive_net)
  cycle_net = osmactive::classify_cycle_infrastructure(cycle_net)
  # filter cycle_net based on column bicycle is yes dismount adn designated
  cycle_net = cycle_net |>
    dplyr::filter(bicycle %in% c("yes", "dismount", "designated")) |>
    dplyr::filter(cycle_segregation == "Separated cycle track") |>
    dplyr::mutate(length = as.numeric(sf::st_length(geometry))) |>
    dplyr::filter(length > length_threshold) |>
    sf::st_transform(crs = 27700)

  snapped_lines = sf::st_snap(cycle_net, cycle_net, tolerance = 15)
  group_ids = sapply(sf::st_geometry(snapped_lines), function(geometry, index, lines) {
    possible_near = sf::st_intersects(geometry, lines, sparse = FALSE)
    connected = which(possible_near)
    unioned = sf::st_union(sf::st_geometry(lines[connected, ]))
    return(unioned)
  }, index = lines_index, lines = snapped_lines)

    # Create a new sf object with merged lines
    merged_lines = sf::st_sf(geometry = do.call(sf::st_sfc, group_ids))
    merged_lines = merged_lines[!duplicated(sf::st_as_text(merged_lines$geometry)), ]    

    network = merged_lines
    network_multilines = network[sf::st_geometry_type(network) == "MULTILINESTRING", ]
    network_lines = sf::st_cast(network_multilines, "LINESTRING")

    network = network_lines
    if (!inherits(network, "sfnetwork")) {
        network_sfn = sfnetworks::as_sfnetwork(network, directed = FALSE)
    } else {
        network_sfn = network
    }

    network_igraph = tidygraph::as_tbl_graph(network_sfn)   

    components = igraph::components(network_igraph)
    component_ids = order(components$csize, decreasing = TRUE)[1:10]  # top 10 components by size

    top_components_sfn = list()  # Initialize list to store sfnetwork components

    # Extract each of the top 10 components and convert them
    for (component_id in component_ids) {
        component_nodes = which(components$membership == component_id)
        component_subgraph = igraph::subgraph(network_igraph, component_nodes)
        if (length(component_nodes) > 0) {  # Check if there are nodes in the component
            component_sfn = sfnetworks::as_sfnetwork(component_subgraph, directed = FALSE)
            top_components_sfn[[component_id]] = component_sfn
        } else {
            top_components_sfn[[component_id]] = NULL
        }
    }

    valid_components = sapply(top_components_sfn, function(x) !is.null(x) && inherits(x, "sfnetwork"))
    all_edges = NULL

    # Loop through each component, activate edges, convert to sf, and combine
    for (i in seq_along(top_components_sfn)) {
        if (!is.null(top_components_sfn[[i]]) && inherits(top_components_sfn[[i]], "sfnetwork")) {
            # Activate edges and convert to sf
            edges_sf = top_components_sfn[[i]] |>
                sfnetworks::activate("edges") |>
                sf::st_as_sf() |>
                dplyr::mutate(component = as.factor(i))  # Add component ID
            
            # Combine into one sf object
            if (is.null(all_edges)) {
                all_edges = edges_sf
            } else {
                all_edges = dplyr::rbind(all_edges, edges_sf)
            }
        }
    }

    sf::st_crs(all_edges) = 27700

    funs = list()

    name_list = names(NPT_zones) 
    for (name in name_list) {
    if (name == "geometry") {
        next  # Correctly skip the current iteration if the name is "geometry"
    } else if (name %in% c("gradient", "quietness")) {
        funs[[name]] = mean  # Assign mean function for specified fields
    } else {
        funs[[name]] = mean  # Assign sum function for all other fields
    }
    }

    filtered_OS_zones = all_edges |> 
                        sf::st_transform(27700) |> 
                        sf::st_zm()
    cycle_net_NPT =  stplanr::rnet_merge(filtered_OS_zones, NPT_zones, dist = 10, funs = funs, max_angle_diff = 10)

    summarized_data = cycle_net_NPT |>
        dplyr::group_by(component) |>
        dplyr::summarize(total_all_fastest_bicycle_go_dutch = sum(all_fastest_bicycle_go_dutch, na.rm = TRUE))

    min_percentile_value = stats::quantile(summarized_data$total_all_fastest_bicycle_go_dutch, probs = 0.3, na.rm = TRUE)
    
    summarized_data = summarized_data |> dplyr::filter(total_all_fastest_bicycle_go_dutch > min_percentile_value)

    return(summarized_data)
}
```