initial commit

Author: Alex Jakubow

.Rprofile

@@ -0,0 +1 @@
+source("renv/activate.R")

.gitignore

@@ -0,0 +1,8 @@
+.Rproj.user
+.Rhistory
+.RData
+.Ruserdata
+
+# Directories
+data/**
+_targets/**

R/functions.R

@@ -0,0 +1,36 @@
+download_source <- function(type = "individual",
+                            outdir) {
+  # Setup for scrape
+  home <- "https://www.ussc.gov"
+  url <- paste0(home, "/research/datafiles/commission-datafiles")
+  all_links_css <- ".subContainer a" 
+  
+  # Determine links
+  links <- read_html(url) %>%
+    html_elements(all_links_css) %>%
+    html_attr("href")
+  if (type == "individual") {
+    regex_str <- "opafy"
+  }
+  links <- paste0(home,
+                 links[grepl("\\.zip$", links) & grepl(regex_str, links)]
+                 )
+  
+  # Create output directory if needed
+  if (!dir.exists(outdir)) {
+    dir.create(outdir, recursive = TRUE, showWarnings = FALSE)
+  }
+  
+  # Download
+  safe_dl <- safely(download.file)
+  map(
+    .x = links,
+    .f = ~ {
+      Sys.sleep(10);
+      safe_dl(.x, 
+              destfile = paste0(outdir, "/", gsub("^.+/", "", .x))
+              )
+    },
+    .progress = TRUE
+    )
+}

_setup.R

@@ -0,0 +1,11 @@
+# Directory setup
+dir.create("R")
+file.create("R/functions.R")
+
+# Install
+install.packages(c("targets", "usethis", "visNetwork"))
+install.packages(c("tidyverse", "rvest"))
+renv::snapshot()
+
+# Initialize pipeline
+targets::use_targets()

_targets.R

@@ -0,0 +1,23 @@
+# Load packages required to define the pipeline
+library(targets)
+
+# Set target options
+tar_option_set(
+  packages = c("tidyverse", "rvest"), # packages that your targets need to run
+  format = "rds" # default storage format
+  # Set other options as needed.
+)
+
+# tar_make_clustermq() configuration (okay to leave alone):
+options(clustermq.scheduler = "multicore")
+
+# Run the R scripts in the R/ folder with your custom functions
+tar_source()
+
+# Pipeline
+list(
+  tar_target(
+    name = source_files,
+    command = download_source(outdir = "data/01_source")
+  )
+)

_targets/.gitignore

@@ -0,0 +1,5 @@
+*
+!.gitignore
+!meta
+meta/*
+!meta/meta

_targets/meta/meta

@@ -0,0 +1,3 @@
+name|type|data|command|depend|seed|path|time|size|bytes|format|repository|iteration|parent|children|seconds|warnings|error
+download_source|function|34e74197f3ce84c5|||||||||||||||
+source_files|stem|a1809566a143e50c|ac3c124c7b3257b9|318b40e636bcae4b|1238641790||t19411.732262596s|21cb808e5a67bfdb|302|rds|local|vector|||942.152|downloaded length 28277929  reported length 34311337. URL httpswww.ussc.govsitesdefaultfileszipopafy19nid.zip Timeout of 60 seconds was reached. downloaded length 30506153  reported length 37178129. URL httpswww.ussc.govsitesdefaultfileszipopafy18nid.zip Timeout of 60 seconds was reached. downloaded length 29965481  reported length 33885462. URL httpswww.ussc.govsitesdefaultfileszipopafy17nid.zip Timeout of 60 seconds was reached. downloaded length 27311273  reported length 34369031. URL httpswww.ussc.govsitesdefaultfileszipopafy16nid.zip Timeout of 60 seconds was reached. downloaded length 32914601  reported length 35341457. URL httpswww.ussc.govsitesdefaultfileszipopafy14nid.zip Timeout of 60 seconds was reached|