01_Scrape_Austrian.Rmd

---
title: "Austrian Proposals and Bills"
output: html_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
```

# Packages

```{r}
pacman::p_load(tidyverse, furrr, rvest, ggthemes, lubridate, rio, haven)

plan(multiprocess)
```

# Getting Index pages for XX - XXV legislative period

```{r}
# Reading stable URLs
XXV <- "https://www.parlament.gv.at/PAKT/VHG/index.shtml?VV=&pageNumber=&ZUAS=ALLE&ALT=&feldRnr=5&FR=ALLE&R_MSFRASZU=MIN&LDAP_GRP(PM_ESM)=N&DR=&FBEZ=FP_001&view=&VHG=GESVOR&VHG2=ALLE&jsMode=&VHG3=ALLE&requestId=C536DCDD28&VHG4=ALLE&MISS=&listeId=100&ALTKN=&filterJq=&AUS=ALLE&SUCH=&DEB=&xdocumentUri=%2FPAKT%2FVHG%2Findex.shtml&R_MFRAS=MIN&GP=XXV&STEP=2905&INTRANET=N&LDAP_GRP(PM_HA)=N&ZUFR=ALLE&NAB=&SID=ALLE&ascDesc=DESC&NRBR=NR&ALTPN=&GBEZ=&AS=ALLE&MIN=ALLE&LISTE=&ZUSTIMM=ALLE&LH=ALLE&GESVOR=ALLE&ZEIT="

XXIV <- "https://www.parlament.gv.at/PAKT/VHG/index.shtml?VV=&pageNumber=&ZUAS=ALLE&ALT=&feldRnr=5&FR=ALLE&R_MSFRASZU=MIN&LDAP_GRP(PM_ESM)=N&DR=&FBEZ=FP_001&view=&VHG=GESVOR&VHG2=ALLE&jsMode=&VHG3=ALLE&requestId=C536DCDD28&VHG4=ALLE&MISS=&listeId=100&ALTKN=&filterJq=&AUS=ALLE&SUCH=&DEB=&xdocumentUri=%2FPAKT%2FVHG%2Findex.shtml&R_MFRAS=MIN&GP=XXIV&STEP=2905&INTRANET=N&LDAP_GRP(PM_HA)=N&ZUFR=ALLE&NAB=&SID=ALLE&ascDesc=DESC&NRBR=NR&ALTPN=&GBEZ=&AS=ALLE&MIN=ALLE&LISTE=&ZUSTIMM=ALLE&LH=ALLE&GESVOR=ALLE&ZEIT="

XXIII <- "https://www.parlament.gv.at/PAKT/VHG/index.shtml?VV=&pageNumber=&ZUAS=ALLE&ALT=&feldRnr=5&FR=ALLE&R_MSFRASZU=MIN&LDAP_GRP(PM_ESM)=N&DR=&FBEZ=FP_001&view=&VHG=GESVOR&VHG2=ALLE&jsMode=&VHG3=ALLE&requestId=C536DCDD28&VHG4=ALLE&MISS=&listeId=100&ALTKN=&filterJq=&AUS=ALLE&SUCH=&DEB=&xdocumentUri=%2FPAKT%2FVHG%2Findex.shtml&R_MFRAS=MIN&GP=XXIII&STEP=2905&INTRANET=N&LDAP_GRP(PM_HA)=N&ZUFR=ALLE&NAB=&SID=ALLE&ascDesc=DESC&NRBR=NR&ALTPN=&GBEZ=&AS=ALLE&MIN=ALLE&LISTE=&ZUSTIMM=ALLE&LH=ALLE&GESVOR=ALLE&ZEIT="

XXII <-"https://www.parlament.gv.at/PAKT/VHG/index.shtml?VV=&pageNumber=&ZUAS=ALLE&ALT=&feldRnr=5&FR=ALLE&R_MSFRASZU=MIN&LDAP_GRP(PM_ESM)=N&DR=&FBEZ=FP_001&view=&VHG=GESVOR&VHG2=ALLE&jsMode=&VHG3=ALLE&requestId=C536DCDD28&VHG4=ALLE&MISS=&listeId=100&ALTKN=&filterJq=&AUS=ALLE&SUCH=&DEB=&xdocumentUri=%2FPAKT%2FVHG%2Findex.shtml&R_MFRAS=MIN&GP=XXII&STEP=2905&INTRANET=N&LDAP_GRP(PM_HA)=N&ZUFR=ALLE&NAB=&SID=ALLE&ascDesc=DESC&NRBR=NR&ALTPN=&GBEZ=&AS=ALLE&MIN=ALLE&LISTE=&ZUSTIMM=ALLE&LH=ALLE&GESVOR=ALLE&ZEIT="

XXI <- "https://www.parlament.gv.at/PAKT/VHG/index.shtml?VV=&pageNumber=&ZUAS=ALLE&ALT=&feldRnr=5&FR=ALLE&R_MSFRASZU=MIN&LDAP_GRP(PM_ESM)=N&DR=&FBEZ=FP_001&view=&VHG=GESVOR&VHG2=ALLE&jsMode=&VHG3=ALLE&requestId=C536DCDD28&VHG4=ALLE&MISS=&listeId=100&ALTKN=&filterJq=&AUS=ALLE&SUCH=&DEB=&xdocumentUri=%2FPAKT%2FVHG%2Findex.shtml&R_MFRAS=MIN&GP=XXI&STEP=2905&INTRANET=N&LDAP_GRP(PM_HA)=N&ZUFR=ALLE&NAB=&SID=ALLE&ascDesc=DESC&NRBR=NR&ALTPN=&GBEZ=&AS=ALLE&MIN=ALLE&LISTE=&ZUSTIMM=ALLE&LH=ALLE&GESVOR=ALLE&ZEIT="

XX <- "https://www.parlament.gv.at/PAKT/VHG/index.shtml?VV=&pageNumber=&ZUAS=ALLE&ALT=&feldRnr=5&FR=ALLE&R_MSFRASZU=MIN&LDAP_GRP(PM_ESM)=N&DR=&FBEZ=FP_001&view=&VHG=GESVOR&VHG2=ALLE&jsMode=&VHG3=ALLE&requestId=C536DCDD28&VHG4=ALLE&MISS=&listeId=100&ALTKN=&filterJq=&AUS=ALLE&SUCH=&DEB=&xdocumentUri=%2FPAKT%2FVHG%2Findex.shtml&R_MFRAS=MIN&GP=XX&STEP=2905&INTRANET=N&LDAP_GRP(PM_HA)=N&ZUFR=ALLE&NAB=&SID=ALLE&ascDesc=DESC&NRBR=NR&ALTPN=&GBEZ=&AS=ALLE&MIN=ALLE&LISTE=&ZUSTIMM=ALLE&LH=ALLE&GESVOR=ALLE&ZEIT="

index_urls <- c(XX, XXI, XXII, XXIII, XXIV, XXV)

```

# Scraping Metadata of indexpages and page URLs

```{r}
# get proposal pages URLs
scrape_indexpage <- function(url) { # URL = XXV, XXIV, XXIII, ...
  
  #read in page
  indexpage <- read_html(url)
  
  proposalpages <- indexpage %>%
    html_nodes("a.link-indicator") %>%
    html_attr("href") %>%
    .[seq(1, length(.), by = 2)] %>%
    paste0("https://www.parlament.gv.at", .)

  # get dates of proposals (website upload?), type (RV, A, VOLKBG, BRA), descriptive title, 
  indexpagedata <- indexpage %>%
    html_nodes("span.table-responsive__inner") %>%
    html_text(trim = T) %>%  # get table inner text and trim
    matrix(ncol = 5, byrow = T) %>%  # put into matrix, then dataframe (tibble)
    .[, -5] %>%
    as_tibble(.name_repair = "minimal") %>%
    setNames(c("web_date", "type", "desc_title", "proposal_id"))

  # add proposal pages URLs
  indexpagedata$proposal_link <- proposalpages
  
  # adding legislative period indicator
  indexpagedata$period <- url %>%
    str_extract("GP=\\w+") %>%
    str_sub(., start = 4)
  
  # add rudimentary status according to status bar (image)
  indexpagedata$status <- indexpage %>%
    html_nodes("img.status") %>%
    html_attr("src") %>%
    .[seq(1, length(.), by = 2)] %>%
    str_extract("[1-9]") %>%
    as.character()
    
    

  # cleaning proposal IDs to be used as filenames
  indexpagedata$proposal_filename <- indexpagedata$proposal_id %>%
    gsub("[^A-Za-z0-9]", "" , .) %>%
    paste(indexpagedata$period, ., sep = "_")
  
  #exporting to appropriatly named .csv
  write.csv(indexpagedata, file = paste0("data/df_", indexpagedata$period[1], ".csv"))
  
}

# scrape_indexpage(XXV)
# scrape_indexpage(XXIV)
# scrape_indexpage(XXIII)
# scrape_indexpage(XXII)
# scrape_indexpage(XXI)
# scrape_indexpage(XX)

```

# merging indexdata dataframes for all legislative periods

```{r}

df_XXV <- read.csv(file = "data/df_XXV.csv", header = TRUE)
df_XXIV <- read.csv(file = "data/df_XXIV.csv", header = TRUE)
df_XXIII <- read.csv(file = "data/df_XXIII.csv", header = TRUE)
df_XXII <- read.csv(file = "data/df_XXII.csv", header = TRUE)
df_XXI <- read.csv(file = "data/df_XXI.csv", header = TRUE)
df_XX <- read.csv(file = "data/df_XX.csv", header = TRUE)

all_indexpagedata <- rbind(df_XXV, df_XXIV, df_XXIII, df_XXII, df_XXI, df_XX)[, -1]

write.csv(all_indexpagedata, file = "data/indexpagedata.csv")

```
# Main

## Scraping from proposal pages

```{r}
# reading in indexpagedata and preparing variables (as.character for strings)
indexpagedata <- as_tibble(read.csv(file = "data/indexpagedata.csv", header = TRUE))
indexpagedata$proposal_link <- as.character(indexpagedata$proposal_link)
indexpagedata$proposal_id <- as.character(indexpagedata$proposal_id)
indexpagedata$proposal_filename <- as.character(indexpagedata$proposal_filename)
indexpagedata$desc_title <- as.character(indexpagedata$desc_title)


#create list for proposalpages
proposalpage <- vector(mode = "list", length = 4544)

for (i in 3659:length(proposalpage)) {
  proposalpage[[i]] <- read_html(indexpagedata$proposal_link[i])
  t <- runif(1, 1.2, 1.4)
  Sys.sleep(t)
  cat(paste0(i, " - waited for ", round(t, 2), "s. "))
}

# create vectors to be used
bill_link <- vector(mode = "character", length = length(proposalpage))
bill_id <- vector(mode = "character", length = length(proposalpage))
resolution_NR <- vector(mode = "list", length = length(proposalpage))
proposal_download <- vector(mode = "character", length = length(proposalpage))
iniator <- vector(mode = "character", length = length(proposalpage))
parl_verfahren <- vector(mode = "list", length = length(proposalpage))
X <- vector("numeric", length = length(proposalpage))

for (i in 1:length(proposalpage)) {
  # bill_link bill_id
  if(is_empty(proposalpage[[i]] %>%
              html_nodes("div.floatLeft p a") %>%
              html_attr("href") %>%
              str_subset("BNR_\\d+")) == 1) {
    bill_link[i] <- NA
  } else {
    bill_link[i] <- proposalpage[[i]] %>%
      html_nodes("div.floatLeft p a") %>%
      html_attr("href") %>%
      str_subset("BNR_\\d+") %>%
      str_replace("\\/pls.+[=]", "https://www.parlament.gv.at")
    bill_id[i] <- bill_link[i] %>%
      str_extract("XX.+BNR_[0-9]*") %>%
      str_replace("\\/B.+_00", "_BNR")
  }
  # resolution_NR
  if(is.na(proposalpage[[i]] %>%
           html_node("div.floatLeft p") %>%
           html_text() %>%
           str_to_lower() %>%
           str_extract("daf.+d") %>%
           gsub("[^A-Za-z0-9]", "" , .) %>%
           str_sub(start = 5L, end = -2L) %>%
           str_split("") %>%
           .[[1]]) == 1) {
    resolution_NR[[i]] <- list(party = NA,
                               pro = NA)
  } else {
    pro <- proposalpage[[i]] %>%
      html_node("div.floatLeft p") %>%
      html_text() %>%
      str_to_lower() %>%
      str_extract("daf.+da") %>%
      gsub("[^A-Za-z0-9]", "" , .) %>%
      str_sub(start = 5L, end = -3L) %>%
      str_split("")
    contra <- proposalpage[[i]] %>%
      html_node("div.floatLeft p") %>%
      html_text() %>%
      str_to_lower() %>%
      str_extract("dag[A-Za-z:\\s,]+") %>%
      gsub("[^A-Za-z0-9]", "" , .) %>%
      str_sub(start = 8L, end = -1L) %>%
      str_split("")
    resolution_NR[[i]] <- list(party = c(pro[[1]], contra[[1]]),
                               pro = c(rep(1, lengths(pro)), rep(0, lengths(contra)))
    )
  }
  # proposal_download
  if(indexpagedata$type[i] == "A") {
    if(is_empty(proposalpage[[i]] %>%
                html_nodes("ul.fliesstext li a") %>%
                html_attr("href") %>%
                str_subset("\\d.html") %>% .[1]) == 1) {
      proposal_download[i] <- NA
    } else {
      proposal_download[i] <- proposalpage[[i]] %>%
        html_nodes("ul.fliesstext li a") %>%
        html_attr("href") %>%
        str_subset("\\d.html") %>% .[1]
    }
  } else if(indexpagedata$type[i] == "RV") {
    if(is_empty(proposalpage[[i]] %>%
                html_nodes("ul.fliesstext li a") %>%
                html_attr("href") %>%
                str_subset("\\d.html") %>% .[1]) == 1) {
      proposal_download[i] <- NA
    } else {
      proposal_download[i] <- proposalpage[[i]] %>%
        html_nodes("ul.fliesstext li a") %>%
        html_attr("href") %>%
        str_subset("\\d.html") %>% .[1]
    }
  } else if(indexpagedata$type[i] == "GABR") {
    if(is_empty(proposalpage[[i]] %>%
                html_nodes("ul.fliesstext li a") %>%
                html_attr("href") %>%
                str_subset("\\d.html") %>% .[1]) == 1) {
      proposal_download[i] <- NA
    } else {
      proposal_download[i] <- proposalpage[[i]] %>%
        html_nodes("ul.fliesstext li a") %>%
        html_attr("href") %>%
        str_subset("\\d.html") %>% .[1]
    }
  } else if(indexpagedata$type[i] == "BUA") {
    if(is_empty(proposalpage[[i]] %>%
                html_nodes("ul.fliesstext li a") %>%
                html_attr("href") %>%
                str_subset("\\d.html") %>% .[2]) == 1) {
      proposal_download[i] <- NA
    } else {
      proposal_download[i] <- proposalpage[[i]] %>%
        html_nodes("ul.fliesstext li a") %>%
        html_attr("href") %>%
        str_subset("\\d.html") %>% .[2]
    }
  }
  # iniator
  if(is_empty(proposalpage[[i]] %>%
              html_nodes("div.c_2 p") %>%
              html_text() %>%
              str_remove_all("[^A-Za-z0-9 ÄÖÜäöü:.-]") %>% .[-1] %>%
              str_extract(":.+") %>%
              str_extract("[A-Z].+") %>%
              str_trim() %>%
              str_subset("[0-9]{2,}", negate = T) %>%
              str_subset("\\s{3,}", negate = T)) == 1) {
    iniator[i] <- NA
  } else {
    iniator[i] <- proposalpage[[i]] %>%
      html_nodes("div.c_2 p") %>%
      html_text() %>%
      str_remove_all("[^A-Za-z0-9 ÄÖÜäöü:.-]") %>% .[-1] %>%
      str_extract(":.+") %>%
      str_extract("[A-Z].+") %>%
      str_trim() %>%
      str_subset("[0-9]{2,}", negate = T) %>%
      str_subset("\\s{3,}", negate = T)
  }
  # add nested df with parl_verfahren
  dates <- proposalpage[[i]] %>%
    html_nodes("table.table-nonresponsive") %>%
    html_nodes("tr.historyShowAlways") %>%
    html_text(trim = T) %>%
    str_subset("[0-9]{2}.[0-9]{2}.[0-9]{4}") %>%
    str_extract("[0-9]{2}.[0-9]{2}.[0-9]{4}")
  
  processes <- proposalpage[[i]] %>%
    html_nodes("a.historieOverviewToggle") %>%
    html_text()
  
  parl_verfahren[[i]] <- list(date = dates,
                              process = processes)
  # add index (X) for merging purposes
  X[i] <- indexpagedata$X[i]
}

proposalpagedata <- tibble(X, bill_link, bill_id, proposal_download, iniator, resolution_NR)

data1 <- left_join(indexpagedata, proposalpagedata, by = "X")

# change NAs to character(0)
for (i in 1:length(data1$proposal_download)) {
  if(is.na(data1$proposal_download[i])) {
    data1$proposal_download[i] <- ""
  }
}

# download loop proposals
for (i in 2437:length(data1$proposal_download)) {
  if(str_detect(data1$proposal_download[i], "^\\/PAKT") == 1) {
    data1$proposal_download[i] %>%
      paste0("www.parlament.gv.at", .) %>%
      download_html(file = paste0("data/proposal/", data1$proposal_filename[i], ".html"))
    t <- runif(1, 1, 1.2)
    Sys.sleep(t)
    cat(paste0(i, " - waited for ", round(t, 2), "s. "))
  }
}

```

## Scraping from bill pages

```{r}

bill_download <- vector(mode = "character", length = length(data1$X))

for (i in 1:length(data1$bill_link)) {
  if(is.na(data1$bill_link[i])) {
    data1$bill_link[i] <- ""
  }
  if(str_detect(data1$bill_link[i], "^\\/PAKT") == 1) {
    data1$bill_link[i] <- paste0("https://www.parlament.gv.at", bill_link[i])
  }
  if(str_detect(data1$bill_link[i], "PAKT") == 1) {
    billpage <- read_html(data1$bill_link[i])
    bill_download[i] <- billpage %>%
      html_nodes("ul.fliesstext li a") %>%
      html_attr("href") %>%
      str_subset("\\d.html") %>% .[1]
  t <- runif(1, 1, 1.2)
  Sys.sleep(t)
  cat(paste0(i, " - waited for ", round(t, 2), "s. "))
  }
}

data1 <- add_column(data1, bill_download)

# change NAs to character(0)
for (i in 1:length(data1$proposal_download)) {
  if(is.na(data1$proposal_download[i])) {
    data1$proposal_download[i] <- ""
  }
}

# download bills (bill_id used as bill_filename)
for (i in 1:length(data1$bill_download)) {
  if(str_detect(data1$bill_download[i], "^\\/PAKT") == 1) {
    data1$bill_download[i] %>%
      paste0("www.parlament.gv.at", .) %>%
      download_html(file = paste0("data/bill/", data1$bill_id[i], ".html"))
    t <- runif(1, 1, 1.2)
    Sys.sleep(t)
    cat(paste0(i, " - waited for ", round(t, 2), "s. "))
  }
}

```