diff --git a/dev/rfp_source_bcdata.R b/dev/rfp_source_bcdata.R new file mode 100644 index 0000000..d93ee6f --- /dev/null +++ b/dev/rfp_source_bcdata.R @@ -0,0 +1,45 @@ +#' Load bcdata layers to a geopackage. +#' +#' This function loads bcdata layers to a `background_layers.gpkg` with the specified watershed groups. The update +#' option can be set to update an existing geopackage if it exists. +#' +#' @param watershed_groups A character vector of watershed group codes. Double quoted list of single quoted values. Default is NULL. +#' @param update A logical indicating whether to update the GeoPackage file. Default is FALSE. +#' +#' @return Invisible NULL. The function is called for its side effects. +#' @importFrom reticulate use_condaenv +#' @export +#' +#' @examples +#' \dontrun{ +#' rfp_source_bcdata(watershed_groups = c("ADMS", "BULK")) +#' } +rfp_source_bcdata <- function(watershed_groups = NULL, update = FALSE) { + # Construct the path to the script within the installed package + scriptPath <- system.file("extdata", "rfp_source_bcdata.sh", package = "rfp") + + # Check if the script exists to avoid errors + if(!file.exists(scriptPath)) { + stop("Script 'rfp_source_bcdata.sh' not found in the package.") + } + + if(update){ + update <- "update" + }else{ + update <- "" + } + + # Convert watershed_groups to a single, comma-separated string + watershed_groups <- paste(shQuote(watershed_groups, type = "sh"), collapse = " ") + + # Prepare the arguments for the script + args <- c(watershed_groups, update) + + # Call the bash script with arguments + system2("bash", args = c(scriptPath, watershed_groups, update)) + + invisible(NULL) +} + + + diff --git a/inst/extdata/rfp_qgis_create_bcfishpass.sh b/inst/extdata/rfp_qgis_create_bcfishpass.sh new file mode 100755 index 0000000..a2789ba --- /dev/null +++ b/inst/extdata/rfp_qgis_create_bcfishpass.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -euxo pipefail + + +# check that the name of the new directory for the project is provided as argument +if [ $# -eq 0 ] + then + echo "No arguments supplied - provide name of new project directory to be placed in ~Projects/gis/" + exit 1 +fi + +echo 'Generating QGIS project in Projects/gis/: '$1 + + +# --------------- +# create directory for project and move the files in and copy in a qlr template (works relative to location dragged from when we reference gpkg) along with the directories for info ignored on phones + +mkdir ~/Projects/gis/$1 +mv background_layers.gpkg* ~/Projects/gis/$1/ +mv habitat_lateral.tif ~/Projects/gis/$1/ +cp ../../data/qgis/bcfishpass_mobile.qgs ~/Projects/gis/$1/$1.qgs +cp ../../data/qgis/form_pscis.gpkg ~/Projects/gis/$1/ +cp ../../data/qgis/form_fiss_site.gpkg ~/Projects/gis/$1/ +mkdir -p ~/Projects/gis/$1/ignore_mobile/photos +touch ~/Projects/gis/$1/ignore_mobile/photos/photos.txt +cp -R ../../data/methods ~/Projects/gis/$1/ignore_mobile/methods +cp -R ../../data/templates ~/Projects/gis/$1/ignore_mobile/templates diff --git a/inst/extdata/rfp_source_aws.sh b/inst/extdata/rfp_source_aws.sh new file mode 100755 index 0000000..bf945b4 --- /dev/null +++ b/inst/extdata/rfp_source_aws.sh @@ -0,0 +1,141 @@ +#!/bin/bash +set -euxo pipefail + +# --------------- +# extract layers from aws to load to background_layers.gpkg within digital field projects + +# usage: specify watershed groups of interest as a comma separated, single quoted string +# eg +# test with ./rfp_sources_aws.sh "'PARS'" &>log.txt +# --------------- + + +# check that watershed group code is provided as argument +if [ $# -eq 0 ] + then + echo "No arguments supplied - provide list of watershed_group_code values for watersheds of interest" + exit 1 +fi + + +# Name of the file +GPKG="background_layers.gpkg" + +# name the sources file +SOURCES="rfp_source_aws.txt" + +# Check if the GPKG IS NOT there +if [[ ! -f $GPKG ]]; then + # If the file exists, ask the user if they wish to remove it + echo "There should already be a file called $GPKG. Please provide." + exit 1 +else +echo 'Updateing project area from watershed group boundaries: '$1 + rm -f aoi.geojson + bcdata dump WHSE_BASEMAPPING.FWA_WATERSHED_GROUPS_POLY \ + --query "WATERSHED_GROUP_CODE in ($1)" -l > aoi.geojson + ogr2ogr -f GPKG background_layers.gpkg \ + -update \ + -overwrite \ + -t_srs EPSG:3005 \ + -nln fwa_watershed_groups_poly \ + aoi.geojson +fi + +# get bounding box of project area in BC Albers and WGS84 (lon/lat) +BOUNDS=$(fio info background_layers.gpkg --layer fwa_watershed_groups_poly --bounds) +BOUNDS_LL=$(echo "[$BOUNDS]" | tr ' ', ',' | rio transform --src_crs EPSG:3005 --dst_crs EPSG:4326 | tr -d '[] ') + + +# --------------- +# lateral habitat +# --------------- +# should be able to pipe $(fio cat aoi.geojson --dst_crs EPSG:3005) to rio mask for a one liner, +# but this works fine +echo 'Clipping lateral habitat tiff to project area' +ogr2ogr -f GeoJSON aoi_alb.geojson -t_srs EPSG:3005 aoi.geojson + +rio mask /vsicurl/https://bcfishpass.s3.us-west-2.amazonaws.com/habitat_lateral.tif \ + habitat_lateral.tif \ + --crop \ + --geojson-mask aoi_alb.geojson + +rm aoi_alb.geojson + +# --------------- +# bcfishpass releated data sources archived to flatgeobuf on s3 for fast retrieval +# (and with watershed group code included, no spatial query for clipping required) +# --------------- +echo 'Getting bcfishpass and supporting layers from s3' + +# get a list of the aws layers to update +FGB_SOURCES=$(grep -v '^#' $SOURCES) + +for layer in $FGB_SOURCES; do + ogr2ogr \ + -f GPKG background_layers.gpkg \ + -update \ + -overwrite \ + -nln $layer \ + -t_srs EPSG:3005 \ + -dim XY \ + -spat $BOUNDS \ + -spat_srs EPSG:3005 \ + -where "watershed_group_code in ($1)" \ + /vsicurl/https://newgraph.s3.us-west-2.amazonaws.com/$layer.fgb \ + $layer +done + +# --------------- +# model parameters +# --------------- +echo 'Getting model parameter info from aws' + +# update the model info used to create the crossings and streams layers +CSVS=("parameters_habitat_method" "parameters_habitat_thresholds") + +for layer in "${CSVS[@]}"; do + ogr2ogr \ + -f GPKG background_layers.gpkg \ + -update \ + -overwrite \ + -nln $layer \ + /vsicurl/https://newgraph.s3.us-west-2.amazonaws.com/$layer.csv +done + +# --------------- +# named streams +# --------------- +echo 'Getting named streams from fwapg feature service' +ogr2ogr -f GPKG background_layers.gpkg \ + -update \ + -overwrite \ + -t_srs EPSG:3005 \ + -nln fwa_named_streams \ + -clipsrc aoi.geojson \ + -clipsrclayer aoi \ + "https://features.hillcrestgeo.ca/fwa/collections/whse_basemapping.fwa_named_streams/items.json?bbox=$BOUNDS_LL" + # "http://www.a11s.one:9000/collections/whse_basemapping.fwa_named_streams/items.json?bbox=$BOUNDS_LL" + + +echo 'Creating a record of the layers that were loaded' +# create a record of the layers that were loaded +# Get the current date and time +TIMESTAMP=$(date '+%Y-%m-%d %H:%M') + +# Header for the CSV file +echo "timestamp,content,watershed_groups,source" > temp.csv + +# Use awk to process the text file line by line +awk -v timestamp="$TIMESTAMP" -v watershed_groups="$1" -v source="$SOURCES" '!/^#/ && NF > 0 {print timestamp "," $0 "," watershed_groups "," source}' $SOURCES >> temp.csv + +# Append the temporary CSV file to the GeoPackage +ogr2ogr -append -f "GPKG" $GPKG temp.csv -nln rfp_tracking + +# Remove the temporary CSV file +rm temp.csv + + + + + diff --git a/inst/extdata/rfp_source_bcdata.sh b/inst/extdata/rfp_source_bcdata.sh new file mode 100755 index 0000000..8d143ba --- /dev/null +++ b/inst/extdata/rfp_source_bcdata.sh @@ -0,0 +1,185 @@ +#!/bin/bash +set -euxo pipefail + +# activate our conda environment +source /opt/homebrew/Caskroom/miniconda/base/etc/profile.d/conda.sh +conda activate dff + + +# Check that watershed group code is provided as an argument +if [ $# -eq 0 ]; then + echo "No arguments supplied - provide list of watershed_group_code values for watersheds of interest." + exit 1 +fi + + +# Temporarily disable the nounset option +set +u + +# Assign a default value to $2 if it's not set +operation=${2:-"generate"} + +# Re-enable the nounset option +set -u + + + +# clean out the old files +rm -f *.geojson +rm -f *.tif + + +# Name of the GeoPackage file +GPKG="background_layers.gpkg" + +# name the sources file +SOURCES="rfp_source_bcdata.txt" + + +# Check for "update" flag and existence of the GeoPackage file +if [[ "$operation" == "update" && ! -f "$GPKG" ]]; then + echo "$GPKG does not exist. Cannot perform update operation." + exit 1 +fi + +# # Check for "update" flag and existence of the GeoPackage file +# if [[ "$2" == "update" && ! -f "$GPKG" ]]; then +# echo "$GPKG does not exist. Cannot perform update operation." +# exit 1 +# fi + +process_geopackage() { + local watersheds="$1" + local operation="${2-}" # Default to empty if not set + + echo "Processing watershed group boundaries: $watersheds" + rm -f aoi.geojson + bcdata dump WHSE_BASEMAPPING.FWA_WATERSHED_GROUPS_POLY \ + --query "WATERSHED_GROUP_CODE in ($watersheds)" -l > aoi.geojson + + local update_option="" + if [[ "$operation" == "update" ]]; then + update_option="-update" + fi + + ogr2ogr -f GPKG "$GPKG" \ + $update_option \ + -overwrite \ + -t_srs EPSG:3005 \ + -nln fwa_watershed_groups_poly \ + aoi.geojson +} + +# Main script logic +if [[ -f "$GPKG" ]]; then + if [[ "$operation" != "update" ]]; then + echo "Error: GeoPackage $GPKG exists but no 'update' flag was set." + exit 1 + else + echo "Updating existing $GPKG with area of interest..." + process_geopackage "$1" "update" + fi +else + if [[ "$operation" == "update" ]]; then + echo "Cannot perform update because $GPKG is missing." + exit 1 + else + echo "Generating a new one..." + process_geopackage "$1" + fi +fi + + +# # Main script logic +# if [[ -f "$GPKG" ]]; then +# read -p "Do you wish to start over with a new $GPKG? If you are updating existing project info say NO (y/n): " answer +# if [[ $answer == "y" ]]; then +# echo "Removing $GPKG..." +# rm -f "$GPKG" +# process_geopackage "$1" +# else +# echo "Updating existing $GPKG with area of interest..." +# process_geopackage "$1" "update" +# fi +# else +# if [[ "$operation" == "update" ]]; then +# # This block should never be reached due to the initial check, but it's here for logical completeness +# echo "Cannot perform update because $GPKG is missing." +# exit 1 +# else +# echo "Generating a new one..." +# process_geopackage "$1" +# fi +# fi + + +# get bounding box of aoi in BC Albers and WGS84 (lon/lat) +BOUNDS_LL=$(fio info aoi.geojson --layer aoi --bounds) +BOUNDS=$(echo "[$BOUNDS_LL]" | tr ' ', ',' | rio transform --src_crs EPSG:4326 --dst_crs EPSG:3005 | tr -d '[] ') + + +# --------------- +# non-fwa bcgw layers +# --------------- +echo 'Getting BC Data Catalogue layers - this may take a while' +BCGW_SOURCES=$(grep -v '^#' $SOURCES) +for layer in $BCGW_SOURCES; do + if [ ! -f ./$layer.geojson ]; then + set -e ; bcdata dump $layer --bounds "$BOUNDS" --bounds-crs EPSG:3005 -l > $layer.geojson + fi +done + +# remove empty fwa tables created by bcdata because sql query will fail later +find . -maxdepth 1 -type f -name "*.fwa_*" -size -50c -delete + + +# get the name of the files (except aoi.geojson) that are left after tiny ones are removed +BCGW_SOURCES_CLEANED=$(find . -maxdepth 1 -type f -name "*.geojson" -a '!' -name "aoi.geojson" -exec basename {} .geojson \;) + + +for layer in $BCGW_SOURCES_CLEANED; do + if [[ $layer == *".fwa_"* ]]; then + ogr2ogr -f GPKG \ + background_layers.gpkg \ + -update \ + -overwrite \ + -t_srs EPSG:3005 \ + -nln $layer \ + -dim XY \ + -where "watershed_group_code in ($1)" \ + $layer.geojson + else + ogr2ogr -f GPKG \ + background_layers.gpkg \ + -update \ + -overwrite \ + -t_srs EPSG:3005 \ + -nln $layer \ + -dim XY \ + -clipsrc aoi.geojson \ + -clipsrclayer aoi \ + $layer.geojson + fi +done + +echo 'Creating a record of the layers that were loaded' +# create a record of the layers that were loaded +# Get the current date and time +TIMESTAMP=$(date '+%Y-%m-%d %H:%M') + +# Header for the CSV file +echo "timestamp,content,watershed_groups,source" > temp.csv + +# Use awk to process the text file line by line +awk -v timestamp="$TIMESTAMP" -v watershed_groups="$1" -v source="$SOURCES" '!/^#/ && NF > 0 {print timestamp "," $0 "," watershed_groups "," source}' $SOURCES >> temp.csv + +# Append the temporary CSV file to the GeoPackage +ogr2ogr -append -f "GPKG" $GPKG temp.csv -nln rfp_tracking + +# Remove the temporary CSV file +rm temp.csv + + +echo 'bcdata layers are now loaded to background_layers.gpkg' + +