Update function docs

Author: jrdnbradford

R/as_list.R

@@ -1,6 +1,4 @@
-
-
-#' Method as.list() for class robotstxt_text
+#' Convert robotstxt_text to list
 #'
 #' @param x class robotstxt_text object to be transformed into list
 #' @param ... further arguments (inherited from \code{base::as.list()})
@@ -17,4 +15,4 @@ as.list.robotstxt_text <-
     res$request   <- attr(x, "request")
 
     res
-  }
+  }

R/fix_url.R

@@ -1,4 +1,4 @@
-#' fix_url
+#' Add http protocal if missing from URL
 #'
 #'
 #' @param url a character string containing a single URL
@@ -10,4 +10,4 @@ fix_url <-
       url <- paste0("http://", url)
     }
     url
-  }
+  }

R/get_robotstxt.R

@@ -1,4 +1,4 @@
-#' downloading robots.txt file
+#' Download a robots.txt file
 #'
 #' @param domain domain from which to download robots.txt file
 #' @param warn warn about being unable to download domain/robots.txt because of

R/get_robotstxt_http_get.R

@@ -1,17 +1,16 @@
-
-#' storage for http request response objects
+#' Storage for http request response objects
 #'
 #' @rdname get_robotstxt_http_get
 #'
 #' @export
 rt_last_http         <- new.env()
 rt_last_http$request <- list()
 
-#' get_robotstxt() worker function to execute HTTP request
+#' Execute HTTP request for get_robotstxt()
 #'
 #' @param ssl_verifypeer either 1 (default) or 0, if 0 it disables SSL peer verification, which
 #'   might help with robots.txt file retrieval
-#' @param domain the domain to get tobots.txt. file for
+#' @param domain the domain to get robots.txt. file for
 #' @param user_agent the user agent to use for HTTP request header
 #'
 #' @export

R/get_robotstxts.R

@@ -1,5 +1,4 @@
-
-#' function to get multiple robotstxt files
+#' Download multiple robotstxt files
 #'
 #' @inheritParams get_robotstxt
 #' @param use_futures Should future::future_lapply be used for possible

R/guess_domain.R

@@ -1,4 +1,4 @@
-#' function guessing domain from path
+#' Guess a domain from path
 #' @param x path aka URL from which to infer domain
 guess_domain <- function(x){
 
@@ -23,4 +23,4 @@ guess_domain <- function(x){
     return(domain)
   }
 
-}
+}

R/http_domain_changed.R

@@ -1,4 +1,4 @@
-#' http_domain_changed
+#' Check if HTTP domain changed
 #'
 #' @param response an httr response object, e.g. from a call to httr::GET()
 #'

R/http_subdomain_changed.R

@@ -1,8 +1,8 @@
-#' http_subdomain_changed
+#' Check if HTTP subdomain changed
 #'
 #' @param response an httr response object, e.g. from a call to httr::GET()
 #'
-#' @return logical of length 1 indicating whether or not any domain change
+#' @return logical of length 1 indicating whether or not any subdomain change
 #'     happened during the HTTP request
 #'
 #'

R/http_was_redirected.R

@@ -1,4 +1,4 @@
-#' http_was_redirected
+#' Check if HTTP redirect occurred
 #'
 #' @param response an httr response object, e.g. from a call to httr::GET()
 #'

R/is_suspect_robotstxt.R

@@ -1,6 +1,6 @@
-#' is_suspect_robotstxt
+#' Check if file is valid / parsable robots.txt file
 #'
-#' function that checks if file is valid / parsable robots.txt file
+#' Function that checks if file is valid / parsable robots.txt file
 #'
 #' @param text content of a robots.txt file provides as character vector
 #'
@@ -26,5 +26,3 @@ is_suspect_robotstxt <- function(text){
   # return default
   return(FALSE)
 }
-
-

R/is_valid_robotstxt.R

@@ -1,6 +1,6 @@
-#' function that checks if file is valid / parsable robots.txt file
+#' Validate if a file is valid / parsable robots.txt file
 #'
-#' @param text content of a robots.txt file provides as character vector
+#' @param text content of a robots.txt file provided as character vector
 #' @param check_strickt_ascii whether or not to check if content does adhere to the specification of RFC to use plain text aka ASCII
 #'
 #' @export
@@ -39,5 +39,3 @@ is_valid_robotstxt <- function(text, check_strickt_ascii = FALSE){
 
   )
 }
-
-

R/null_to_default.R

@@ -1,4 +1,4 @@
-#' null_to_default
+#' Return default value if NULL
 #'
 #' @param x value to check and return
 #' @param d value to return in case x is NULL

R/parse_robotstxt.R

@@ -1,9 +1,8 @@
-#' function parsing robots.txt
-#' @param  txt content of the robots.txt file
+#' Parse a robots.txt file
+#' @param txt content of the robots.txt file
 #' @return a named list with useragents, comments, permissions, sitemap
 #' @export
 parse_robotstxt <- function(txt){
-  # return
   res <-
     list(
       useragents  = rt_get_useragent(txt),
@@ -21,21 +20,3 @@ parse_robotstxt <- function(txt){
     )
   return(res)
 }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

R/parse_url.R

@@ -1,13 +1,9 @@
-
-
-
-#' parse_url
+#' Parse a URL
 #'
 #' @param url url to parse into its components
 #'
 #' @return data.frame with columns protocol, domain, path
 #'
-#'
 #' @keywords internal
 #'
 #' @examples

R/paths_allowed.R

@@ -1,12 +1,12 @@
-#' check if a bot has permissions to access page(s)
+#' Check if a bot has permissions to access page(s)
 #'
 #'
 #' @param domain Domain for which paths should be checked. Defaults to "auto".
 #'   If set to "auto" function will try to guess the domain by parsing the paths
 #'   argument. Note however, that these are educated guesses which might utterly
 #'   fail. To be on the safe side, provide appropriate domains manually.
 #' @param bot name of the bot, defaults to "*"
-#' @param paths paths for which to check bot's permission, defaults to "/". Please, note that path to a folder should end with a trailing slash ("/").
+#' @param paths paths for which to check bot's permission, defaults to "/". Please note that path to a folder should end with a trailing slash ("/").
 #' @param check_method at the moment only kept for backward compatibility reasons - do not use parameter anymore --> will let the function simply use the default
 #' @param robotstxt_list either NULL -- the default -- or a list of character
 #'                       vectors with one vector per path to check

R/paths_allowed_worker_spiderbar.R

@@ -1,6 +1,4 @@
-
-
-#' paths_allowed_worker spiderbar flavor
+#' Check if a spiderbar bot has permissions to access page(s)
 #'
 #' @inheritParams paths_allowed
 #'

R/print_robotstxt.R

@@ -1,4 +1,4 @@
-#' printing robotstxt
+#' Print robotstxt
 #' @param x robotstxt instance to be printed
 #' @param ... goes down the sink
 #' @export
@@ -29,4 +29,3 @@ print.robotstxt <- function(x, ...){
   invisible(x)
 }
 
-

R/print_robotstxt_text.R

@@ -1,11 +1,10 @@
-
-#' printing robotstxt_text
+#' Print robotstxt's text
 #' @param x character vector aka robotstxt$text to be printed
 #' @param ... goes down the sink
 #' @export
 print.robotstxt_text <- function(x, ...){
 
-  # rpint part of the robots.txt file
+  # print part of the robots.txt file
   cat("[robots.txt]\n--------------------------------------\n\n")
   tmp <- unlist(strsplit(x, "\n"))
   cat(tmp[seq_len(min(length(tmp), 50))], sep ="\n")
@@ -29,4 +28,4 @@ print.robotstxt_text <- function(x, ...){
 
   # return
   invisible(x)
-}
+}

R/remove_domain.R

@@ -1,4 +1,4 @@
-#' function to remove domain from path
+#' Remove domain from path
 #' @param x path aka URL from which to first infer domain and then remove it
 remove_domain <- function(x){
   unlist(lapply(
@@ -12,5 +12,3 @@ remove_domain <- function(x){
     }
   ))
 }
-
-

R/request_handler_handler.R

@@ -1,10 +1,10 @@
-#' request_handler_handler
+#' Handle robotstxt handlers
 #'
 #' Helper function to handle robotstxt handlers.
 #'
 #' @param request the request object returned by call to httr::GET()
 #' @param handler the handler either a character string entailing various options or a function producing a specific list, see return.
-#' @param res a list a list with elements '[handler names], ...', 'rtxt', and 'cache'
+#' @param res a list with elements '[handler names], ...', 'rtxt', and 'cache'
 #' @param info info to add to problems list
 #' @param warn if FALSE warnings and messages are suppressed
 #'
@@ -72,4 +72,4 @@ request_handler_handler <-
 
     # return
     res
-  }
+  }

R/robotstxt.R

@@ -1,11 +1,12 @@
-#' Generate a representations of a robots.txt file
+#' Generate a representation of a robots.txt file
 #'
 #' The function generates a list that entails data resulting from parsing a robots.txt file
 #' as well as a function called check that enables to ask the representation if bot (or
 #' particular bots) are allowed to access a resource on the domain.
 #'
 #' @param domain Domain for which to generate a representation. If text equals to NULL,
 #' the function will download the file from server - the default.
+#'
 #' @param text If automatic download of the robots.txt is not preferred, the text can be
 #' supplied directly.
 #' @inheritParams get_robotstxt
@@ -20,7 +21,7 @@
 #' @field domain character vector holding domain name for which the robots.txt
 #'   file is valid; will be set to NA if not supplied on initialization
 #'
-#' @field text character vector of text of robots.txt file; either supplied on
+#' @field character vector of text of robots.txt file; either supplied on
 #'   initialization or automatically downloaded from domain supplied on
 #'   initialization
 #'

R/rt_cache.R

@@ -1,3 +1,2 @@
-#' get_robotstxt() cache
+#' Get the robotstxt cache
 rt_cache <- new.env( parent = emptyenv() )
-

R/rt_get_comments.R

@@ -1,5 +1,4 @@
-
-#' extracting comments from robots.txt
+#' Extract comments from robots.txt
 #' @param txt content of the robots.txt file
 #' @keywords internal
 rt_get_comments <- function(txt){
@@ -8,4 +7,3 @@ rt_get_comments <- function(txt){
   ccontent <- stringr::str_extract(txt[clines], "#.*")
   data.frame(line=clines, comment=ccontent, stringsAsFactors = FALSE)
 }
-

R/rt_get_fields.R

@@ -1,5 +1,4 @@
-
-#' extracting permissions from robots.txt
+#' Extract permissions from robots.txt
 #' @param txt content of the robots.txt file
 #' @param regex regular expression specify field
 #' @param invert invert selection made via regex?

R/rt_get_fields_worker.R

@@ -1,5 +1,4 @@
-
-#' extracting robotstxt fields
+#' Extract robotstxt fields
 #' @param txt content of the robots.txt file
 #' @param type name or names of the fields to be returned, defaults to all
 #'   fields

R/rt_get_useragent.R

@@ -1,4 +1,4 @@
-#' extracting HTTP useragents from robots.txt
+#' Extract HTTP useragents from robots.txt
 #' @param txt content of the robots.txt file
 #' @keywords internal
 # rt_get_useragent <- function(txt){
@@ -19,4 +19,4 @@ rt_get_useragent <- function(txt){
     pattern     = stringr::regex("U.*:| |\n", ignore_case = TRUE),
     replacement = ""
   )
-}
+}

R/rt_request_handler.R

@@ -1,15 +1,13 @@
-
-#' rt_request_handler
+#' Handle robotstxt object retrieved from HTTP request
 #'
 #' A helper function for get_robotstxt() that will extract the robots.txt file
-#' from the HTTP request result object. furthermore it will inform
-#' get_robotstxt() if the request should be cached and which problems occured.
+#' from the HTTP request result object. It will inform get_robotstxt() if the
+#' request should be cached and which problems occurred.
 #'
 #'
 #'
 #' @param request result of an HTTP request (e.g. httr::GET())
 #'
-#'
 #' @param on_server_error request state handler for any 5xx status
 #'
 #' @param on_client_error request state handler for any 4xx HTTP status that is
@@ -31,8 +29,8 @@
 #' @param on_suspect_content request state handler for content that seems to be
 #'   something else than a robots.txt file (usually a JSON, XML or HTML)
 #'
-#'
 #' @param warn suppress warnings
+#'
 #' @param encoding The text encoding to assume if no encoding is provided in the
 #'   headers of the response
 #'
@@ -99,8 +97,6 @@ rt_request_handler <-
     }
 
 
-
-
     ## server error
     server_error <-
       request$status_code >= 500
@@ -207,8 +203,6 @@ rt_request_handler <-
     }
 
 
-
-
     ## file type mismatch
     file_type_mismatch <-
       !(grepl("text/plain", null_to_default(request$headers$`content-type`, "")))

R/sanitize_path.R

@@ -1,4 +1,4 @@
-#' making paths uniform
+#' Make paths uniform
 #' @param path path to be sanitized
 #' @return sanitized path
 #' @keywords internal
@@ -7,4 +7,3 @@ sanitize_path <- function(path){
   path <- ifelse( !grepl("^/", path),  paste0("/", path), path)
   return(path)
 }
-