From c4e35e6658c3eb078bb98d2c3defc7f41f9ebcf3 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Sun, 26 May 2024 11:08:55 +0800 Subject: [PATCH 1/4] WIP upgrade dbplyr * Follow upgrade guide at https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition * DESCRIPTION: Bump version (suggest 0.7.1 once merged), bump min versions for jsonlite, dplyr, dbplyr to latest available, update author email for FM, bump RoxygenNote * Fix Roxygen warning: Upgrade syntax in ckanr-package.R * Add dbplyr_edition as per upgrade guide (new man page) * Upgrade dplyr.R generics and imports as per upgrade guide * Document package to rebuild NAMESPACE * TODO NEWS * TODO see if tests pass (no working CKAN available here) * This PR should be reviewed thoroughly before merging --- DESCRIPTION | 12 +++++----- NAMESPACE | 20 +++++++++------- R/ckanr-package.R | 4 +++- R/dplyr.R | 31 +++++++++++++++---------- ckanr.Rproj | 1 + man/ckanr-package.Rd | 12 ++++++++++ man/dbplyr_edition.myConnectionClass.Rd | 15 ++++++++++++ 7 files changed, 67 insertions(+), 28 deletions(-) create mode 100644 man/dbplyr_edition.myConnectionClass.Rd diff --git a/DESCRIPTION b/DESCRIPTION index f106708..a4ae330 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -3,7 +3,7 @@ Title: Client for the Comprehensive Knowledge Archive Network ('CKAN') API Description: Client for 'CKAN' API (). Includes interface to 'CKAN' 'APIs' for search, list, show for packages, organizations, and resources. In addition, provides an interface to the 'datastore' API. -Version: 0.7.0.9000 +Version: 0.7.0.9001 Authors@R: c( person("Scott", "Chamberlain", role = c("aut"), @@ -17,7 +17,7 @@ Authors@R: c( comment = c(ORCID = "0000-0001-5180-0567")), person("Florian", "Mayer", role = "aut", - email="Florian.Mayer@dbca.wa.gov.au", + email="Florian.Mayer@dpc.wa.gov.au", comment = c(ORCID = "0000-0003-4269-4242")), person("Sharla", "Gelfand", role = "aut"), @@ -40,9 +40,9 @@ Imports: stats, utils, crul, - jsonlite (>= 0.9.17), - dplyr (>= 0.7.0), - dbplyr, + jsonlite (>= 1.8.8), + dplyr (>= 1.1.4), + dbplyr (>= 2.5.0), magrittr Suggests: sf, @@ -50,7 +50,7 @@ Suggests: testthat, xml2, lazyeval -RoxygenNote: 7.2.3 +RoxygenNote: 7.3.1 X-schema.org-keywords: database, open-data, ckan, api, data, dataset X-schema.org-applicationCategory: Data Access X-schema.org-isPartOf: "https://ropensci.org" diff --git a/NAMESPACE b/NAMESPACE index 27e5a5a..506d401 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -22,11 +22,10 @@ S3method(as.ckan_user,character) S3method(as.ckan_user,ckan_user) S3method(as.ckan_user,list) S3method(db_begin,CKANConnection) -S3method(db_explain,CKANConnection) S3method(db_has_table,CKANConnection) S3method(db_insert_into,CKANConnection) -S3method(db_query_fields,CKANConnection) S3method(db_query_rows,CKANConnection) +S3method(dbplyr_edition,myConnectionClass) S3method(format,src_ckan) S3method(print,ckan_group) S3method(print,ckan_organization) @@ -36,8 +35,10 @@ S3method(print,ckan_resource) S3method(print,ckan_tag) S3method(print,ckan_user) S3method(print,ckanr_settings) -S3method(sql_translate_env,CKANConnection) -S3method(sql_translate_env,src_ckan) +S3method(sql_query_explain,CKANConnection) +S3method(sql_query_fields,CKANConnection) +S3method(sql_translation,CKANConnection) +S3method(sql_translation,src_ckan) S3method(src_tbls,src_ckan) S3method(tbl,src_ckan) export("%>%") @@ -133,23 +134,24 @@ importFrom(dbplyr,base_agg) importFrom(dbplyr,base_scalar) importFrom(dbplyr,base_win) importFrom(dbplyr,build_sql) +importFrom(dbplyr,dbplyr_edition) importFrom(dbplyr,sql_prefix) +importFrom(dbplyr,sql_query_explain) +importFrom(dbplyr,sql_query_fields) +importFrom(dbplyr,sql_query_select) +importFrom(dbplyr,sql_query_wrap) +importFrom(dbplyr,sql_translation) importFrom(dbplyr,sql_translator) importFrom(dbplyr,sql_variant) importFrom(dbplyr,src_sql) importFrom(dbplyr,tbl_sql) importFrom(dplyr,db_begin) importFrom(dplyr,db_desc) -importFrom(dplyr,db_explain) importFrom(dplyr,db_has_table) importFrom(dplyr,db_insert_into) importFrom(dplyr,db_list_tables) -importFrom(dplyr,db_query_fields) importFrom(dplyr,db_query_rows) importFrom(dplyr,sql) -importFrom(dplyr,sql_select) -importFrom(dplyr,sql_subquery) -importFrom(dplyr,sql_translate_env) importFrom(dplyr,src_tbls) importFrom(dplyr,tbl) importFrom(jsonlite,fromJSON) diff --git a/R/ckanr-package.R b/R/ckanr-package.R index a990b79..3d304b1 100644 --- a/R/ckanr-package.R +++ b/R/ckanr-package.R @@ -71,8 +71,10 @@ #' @author Florian Mayer \email{florian.wendelin.mayer@@gmail.com} #' @author Wush Wu #' @author Imanuel Costigan \email{i.costigan@@me.com} +#' @author Sharla Gelfand +#' @author Francisco Alves \email{fjunior.alves.oliveira@@gmail.com} #' @keywords package -NULL +"_PACKAGE" #' Deprecated functions in \pkg{ckanr} #' diff --git a/R/dplyr.R b/R/dplyr.R index 91e7fec..c6b402d 100644 --- a/R/dplyr.R +++ b/R/dplyr.R @@ -78,9 +78,8 @@ format.src_ckan <- function(x, ...) { } #' @export -#' @importFrom dplyr sql_translate_env #' @importFrom dbplyr base_agg build_sql -sql_translate_env.src_ckan <- function(con) { +sql_translation.src_ckan <- function(con) { sql_variant( base_scalar, sql_translator(.parent = base_agg, @@ -100,8 +99,8 @@ sql_translate_env.src_ckan <- function(con) { } #' @export -sql_translate_env.CKANConnection <- function(con) { - sql_translate_env.src_ckan(con) +sql_translation.CKANConnection <- function(con) { + sql_translation.src_ckan(con) } #' @export @@ -118,8 +117,8 @@ db_begin.CKANConnection <- function(con, ...) { # http://www.postgresql.org/docs/9.3/static/sql-explain.html #' @export -#' @importFrom dplyr db_explain -db_explain.CKANConnection <- function(con, sql, format = "text", ...) { +#' @importFrom dbplyr sql_query_explain +sql_query_explain.CKANConnection <- function(con, sql, format = "text", ...) { format <- match.arg(format, c("text", "json", "yaml", "xml")) exsql <- build_sql("EXPLAIN ", @@ -137,9 +136,8 @@ db_insert_into.CKANConnection <- function(con, table, values, ...) { } #' @export -#' @importFrom dplyr db_query_fields -db_query_fields.CKANConnection <- function(con, sql, ...) { - sql <- sql_select(con, sql("*"), sql_subquery(con, sql), where = sql("0 = 1")) +sql_query_fields.CKANConnection <- function(con, sql, ...) { + sql <- sql_query_select(con, sql("*"), sql_query_wrap(con, sql), where = sql("0 = 1")) qry <- dbSendQuery(con, sql) on.exit(dbClearResult(qry)) @@ -150,13 +148,22 @@ db_query_fields.CKANConnection <- function(con, sql, ...) { #' @export #' @importFrom dplyr db_query_rows db_query_rows.CKANConnection <- function(con, sql, ...) { - from <- sql_subquery(con, sql, "master") + from <- sql_query_wrap(con, sql, "master") # rows <- build_sql("SELECT count(*) FROM ", from, con = con) rows <- sprintf("SELECT count(*) FROM (%s)", unclass(sql)) as.integer(dbGetQuery(con$con, rows)[[1]]) } -#' @importFrom dplyr db_list_tables sql sql_select sql_subquery -#' @importFrom dbplyr base_agg base_scalar base_win build_sql sql_prefix +#' Use dbplyr 2.0.0 generics +#' See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition +#' New in 0.7.0.9001 +#' @importFrom dbplyr dbplyr_edition +#' @export +dbplyr_edition.myConnectionClass <- function(con) 2L + +#' @importFrom dplyr db_list_tables sql +#' @importFrom dbplyr base_agg base_scalar base_win build_sql sql_prefix #' sql_translator sql_variant src_sql tbl_sql +#' sql_query_fields +#' sql_query_select sql_query_wrap sql_query_explain sql_translation NULL diff --git a/ckanr.Rproj b/ckanr.Rproj index 21a4da0..eaa6b81 100644 --- a/ckanr.Rproj +++ b/ckanr.Rproj @@ -15,3 +15,4 @@ LaTeX: pdfLaTeX BuildType: Package PackageUseDevtools: Yes PackageInstallArgs: --no-multiarch --with-keep.source +PackageRoxygenize: rd,collate,namespace diff --git a/man/ckanr-package.Rd b/man/ckanr-package.Rd index a6cf9cd..f946edb 100644 --- a/man/ckanr-package.Rd +++ b/man/ckanr-package.Rd @@ -76,6 +76,14 @@ using this package. That is, not necessarily landing pages of each instance, although, the URL may be the landing page and the base API URL. } +\seealso{ +Useful links: +\itemize{ + \item \url{https://docs.ropensci.org/ckanr/ (website) https://github.com/ropensci/ckanr (devel)} + \item Report bugs at \url{https://github.com/ropensci/ckanr/issues} +} + +} \author{ Scott Chamberlain \email{myrmecocystus@gmail.com} @@ -84,5 +92,9 @@ Florian Mayer \email{florian.wendelin.mayer@gmail.com} Wush Wu Imanuel Costigan \email{i.costigan@me.com} + +Sharla Gelfand + +Francisco Alves \email{fjunior.alves.oliveira@gmail.com} } \keyword{package} diff --git a/man/dbplyr_edition.myConnectionClass.Rd b/man/dbplyr_edition.myConnectionClass.Rd new file mode 100644 index 0000000..d7ed221 --- /dev/null +++ b/man/dbplyr_edition.myConnectionClass.Rd @@ -0,0 +1,15 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/dplyr.R +\name{dbplyr_edition.myConnectionClass} +\alias{dbplyr_edition.myConnectionClass} +\title{Use dbplyr 2.0.0 generics +See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition +New in 0.7.0.9001} +\usage{ +\method{dbplyr_edition}{myConnectionClass}(con) +} +\description{ +Use dbplyr 2.0.0 generics +See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition +New in 0.7.0.9001 +} From e1aac3f396f62704eaa8f85e0f3da754d0decce5 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Sun, 26 May 2024 13:47:32 +0800 Subject: [PATCH 2/4] Add NEWS * Improve DESCRIPTION: add R dependency (prepare to use native pipe over magrittr pipe), pin crul version * Improve ckanr-package author email * Fix dbplyr_edition generics * Suggest new package version 0.8.0 --- DESCRIPTION | 6 +++-- NAMESPACE | 3 ++- NEWS.md | 25 ++++++++++++++++++- R/ckanr-package.R | 2 +- R/dplyr.R | 14 ++++++++--- man/ckanr-package.Rd | 2 +- ...ss.Rd => dbplyr_edition.CKANConnection.Rd} | 10 ++++---- man/dbplyr_edition.src_ckan.Rd | 15 +++++++++++ 8 files changed, 62 insertions(+), 15 deletions(-) rename man/{dbplyr_edition.myConnectionClass.Rd => dbplyr_edition.CKANConnection.Rd} (64%) create mode 100644 man/dbplyr_edition.src_ckan.Rd diff --git a/DESCRIPTION b/DESCRIPTION index a4ae330..004a2bb 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -34,12 +34,14 @@ BugReports: https://github.com/ropensci/ckanr/issues Encoding: UTF-8 Language: en-US Roxygen: list(markdown = TRUE) -Depends: DBI (>= 0.3.1) +Depends: + DBI (>= 1.2.2), + R (>= 4.4.0) Imports: methods, stats, utils, - crul, + crul (>= 1.4.2), jsonlite (>= 1.8.8), dplyr (>= 1.1.4), dbplyr (>= 2.5.0), diff --git a/NAMESPACE b/NAMESPACE index 506d401..23fb8f6 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -25,7 +25,8 @@ S3method(db_begin,CKANConnection) S3method(db_has_table,CKANConnection) S3method(db_insert_into,CKANConnection) S3method(db_query_rows,CKANConnection) -S3method(dbplyr_edition,myConnectionClass) +S3method(dbplyr_edition,CKANConnection) +S3method(dbplyr_edition,src_ckan) S3method(format,src_ckan) S3method(print,ckan_group) S3method(print,ckan_organization) diff --git a/NEWS.md b/NEWS.md index 481b005..d730719 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,28 @@ # ckanr (development version) +ckanr 0.8.0 +=========== + +### NEW FEATURES + +* upgrades deprecated `dbplyr` generics (#187, @florianm) + + Following [dbplyr's 2.0.0 backend API](https://dbplyr.tidyverse.org/articles/backend-2.html), + scripts using `ckanr`'s SQL `CKANConnection` must implement the following + changes: + + * change `dplyr::sql_translate_env` to `dbplyr::sql_translation` + * change `dplyr::db_explain` to `dbplyr::sql_query_explain` + * change `dplyr::db_query_fields` to `dbplyr::sql_query_fields` + * change `dplyr::sql_subquery` to `dbplyr::sql_query_wrap` + * change `dplyr::sql_select` to `dbplyr::sql_query_select` + * all other generics are unchanged. +* updates dependencies (R, DBI, jsonlite, dplyr, dbplyr, RoxygenNote) + +### MINOR IMPROVEMENTS + +* update manpage author list (add SG, FA), author email (FM) + ckanr 0.7.0 =========== @@ -81,7 +104,7 @@ ckanr 0.3.0 * `package_show()` gains `key` parameter to pass an API key (#97) * `package_search()` gains new parameters: `include_drafts`, `include_private`, `use_default_schema`, and `facet.mincount` (#107) * function `fetch()` changed to `ckan_fetch()` -* gains function `organization_delet()` to delete an organization (#83) +* gains function `organization_delete()` to delete an organization (#83) * gains function `ckan_version()` to get version info for a CKAN instance * gains methods for creating a CKAN remote instance as a dplyr backend: gains `src_ckan()` and it's s3 methods `tbl` and `src_tbls`, `sql_translate_env`. in addition gains the S3 methods `db_begin`, `db_explain`, `db_has_table`, `db_insert_into`, `db_query_fields`, `db_query_rows` diff --git a/R/ckanr-package.R b/R/ckanr-package.R index 3d304b1..d313861 100644 --- a/R/ckanr-package.R +++ b/R/ckanr-package.R @@ -68,7 +68,7 @@ #' @aliases ckanr #' @docType package #' @author Scott Chamberlain \email{myrmecocystus@@gmail.com} -#' @author Florian Mayer \email{florian.wendelin.mayer@@gmail.com} +#' @author Florian Mayer \email{florian.wendelin.mayer+ckanr@@gmail.com} #' @author Wush Wu #' @author Imanuel Costigan \email{i.costigan@@me.com} #' @author Sharla Gelfand diff --git a/R/dplyr.R b/R/dplyr.R index c6b402d..60d33ed 100644 --- a/R/dplyr.R +++ b/R/dplyr.R @@ -156,14 +156,20 @@ db_query_rows.CKANConnection <- function(con, sql, ...) { #' Use dbplyr 2.0.0 generics #' See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition -#' New in 0.7.0.9001 +#' Introduced in 0.8.0 #' @importFrom dbplyr dbplyr_edition #' @export -dbplyr_edition.myConnectionClass <- function(con) 2L +dbplyr_edition.CKANConnection <- function(con) 2L + +#' Use dbplyr 2.0.0 generics +#' See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition +#' Introduced in 0.8.0 +#' @importFrom dbplyr dbplyr_edition +#' @export +dbplyr_edition.src_ckan <- function(con) 2L #' @importFrom dplyr db_list_tables sql #' @importFrom dbplyr base_agg base_scalar base_win build_sql sql_prefix #' sql_translator sql_variant src_sql tbl_sql -#' sql_query_fields -#' sql_query_select sql_query_wrap sql_query_explain sql_translation +#' sql_query_fields sql_query_select sql_query_wrap sql_query_explain sql_translation NULL diff --git a/man/ckanr-package.Rd b/man/ckanr-package.Rd index f946edb..3a892c6 100644 --- a/man/ckanr-package.Rd +++ b/man/ckanr-package.Rd @@ -87,7 +87,7 @@ Useful links: \author{ Scott Chamberlain \email{myrmecocystus@gmail.com} -Florian Mayer \email{florian.wendelin.mayer@gmail.com} +Florian Mayer \email{florian.wendelin.mayer+ckanr@gmail.com} Wush Wu diff --git a/man/dbplyr_edition.myConnectionClass.Rd b/man/dbplyr_edition.CKANConnection.Rd similarity index 64% rename from man/dbplyr_edition.myConnectionClass.Rd rename to man/dbplyr_edition.CKANConnection.Rd index d7ed221..0bf2917 100644 --- a/man/dbplyr_edition.myConnectionClass.Rd +++ b/man/dbplyr_edition.CKANConnection.Rd @@ -1,15 +1,15 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/dplyr.R -\name{dbplyr_edition.myConnectionClass} -\alias{dbplyr_edition.myConnectionClass} +\name{dbplyr_edition.CKANConnection} +\alias{dbplyr_edition.CKANConnection} \title{Use dbplyr 2.0.0 generics See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition -New in 0.7.0.9001} +Introduced in 0.8.0} \usage{ -\method{dbplyr_edition}{myConnectionClass}(con) +\method{dbplyr_edition}{CKANConnection}(con) } \description{ Use dbplyr 2.0.0 generics See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition -New in 0.7.0.9001 +Introduced in 0.8.0 } diff --git a/man/dbplyr_edition.src_ckan.Rd b/man/dbplyr_edition.src_ckan.Rd new file mode 100644 index 0000000..adc8873 --- /dev/null +++ b/man/dbplyr_edition.src_ckan.Rd @@ -0,0 +1,15 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/dplyr.R +\name{dbplyr_edition.src_ckan} +\alias{dbplyr_edition.src_ckan} +\title{Use dbplyr 2.0.0 generics +See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition +Introduced in 0.8.0} +\usage{ +\method{dbplyr_edition}{src_ckan}(con) +} +\description{ +Use dbplyr 2.0.0 generics +See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition +Introduced in 0.8.0 +} From bbe14b080ccc3ed86fdfc04615f292bdae340a50 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Wed, 29 May 2024 14:55:32 +0800 Subject: [PATCH 3/4] Patch tests to pass with CKAN 2.10 * See #215: support multiple CKAN versions * Disable tests for deprecated API endpoints as of CKAN 2.10 * Introduce a default for get_test_url and get_test_behaviour * Format ckanr_settings --- .github/workflows/R-check.yaml | 2 - R/ckanr_settings.R | 102 ++++++++++++-------- man/ckanr_setup.Rd | 22 +++-- tests/testthat/test-changes.R | 66 ++++++------- tests/testthat/test-ckan_fetch.R | 10 +- tests/testthat/test-organization_show.R | 8 +- tests/testthat/test-package_activity_list.R | 87 ++++++++--------- tests/testthat/test-tag_show.R | 4 +- 8 files changed, 170 insertions(+), 131 deletions(-) diff --git a/.github/workflows/R-check.yaml b/.github/workflows/R-check.yaml index 49bb44b..7be7257 100644 --- a/.github/workflows/R-check.yaml +++ b/.github/workflows/R-check.yaml @@ -21,8 +21,6 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - CKANR_TEST_URL: ${{secrets.CKANR_TEST_URL}} - CKANR_TEST_BEHAVIOUR: ${{secrets.CKANR_TEST_BEHAVIOUR}} R_REMOTES_NO_ERRORS_FROM_WARNINGS: true R_KEEP_PKG_SOURCE: yes CKAN_VERSION: ${{ matrix.config.ckan-version }} # for docker compose up diff --git a/R/ckanr_settings.R b/R/ckanr_settings.R index ae77680..1e67ef2 100644 --- a/R/ckanr_settings.R +++ b/R/ckanr_settings.R @@ -21,16 +21,17 @@ assign("ckanr_proxy", NULL, envir = ckanr_settings_env) #' @examples #' ckanr_settings() ckanr_settings <- function() { - ops <- list(url = Sys.getenv("CKANR_DEFAULT_URL", ""), - key = Sys.getenv("CKANR_DEFAULT_KEY", ""), - test_url = Sys.getenv("CKANR_TEST_URL", ""), - test_key = Sys.getenv("CKANR_TEST_KEY", ""), - test_did = Sys.getenv("CKANR_TEST_DID", ""), - test_rid = Sys.getenv("CKANR_TEST_RID", ""), - test_gid = Sys.getenv("CKANR_TEST_GID", ""), - test_oid = Sys.getenv("CKANR_TEST_OID", ""), - test_behaviour = Sys.getenv("CKANR_TEST_BEHAVIOUR", ""), - proxy = get("ckanr_proxy", ckanr_settings_env) + ops <- list( + url = Sys.getenv("CKANR_DEFAULT_URL", ""), + key = Sys.getenv("CKANR_DEFAULT_KEY", ""), + test_url = Sys.getenv("CKANR_TEST_URL", ""), + test_key = Sys.getenv("CKANR_TEST_KEY", ""), + test_did = Sys.getenv("CKANR_TEST_DID", ""), + test_rid = Sys.getenv("CKANR_TEST_RID", ""), + test_gid = Sys.getenv("CKANR_TEST_GID", ""), + test_oid = Sys.getenv("CKANR_TEST_OID", ""), + test_behaviour = Sys.getenv("CKANR_TEST_BEHAVIOUR", ""), + proxy = get("ckanr_proxy", ckanr_settings_env) ) structure(ops, class = "ckanr_settings") } @@ -64,7 +65,12 @@ print.ckanr_settings <- function(x, ...) { #' @export #' @param url A CKAN URL (optional), default: https://data.ontario.ca #' @param key A CKAN API key (optional, character) -#' @param test_url (optional, character) A valid CKAN URL for testing purposes +#' @param test_url (optional, character) A valid CKAN URL for testing purposes. +#' Default: "http://localhost:5000". +#' The default works with a local installation of CKAN via docker compose +#' running on port 5000, as well as for CI on GitHub. +#' This enables contributors to run the CI test suite on GitHub when submitting +#' a pull request. #' @param test_key (optional, character) A valid CKAN API key privileged to #' create datasets at `test_url` #' @param test_did (optional, character) A valid CKAN dataset ID, existing at @@ -76,7 +82,8 @@ print.ckanr_settings <- function(x, ...) { #' `test_url` #' @param test_behaviour (optional, character) Whether to fail ("FAIL") or skip #' ("SKIP") writing tests in case of problems with the configured test CKAN. -#' @param proxy an object of class `request` from a call to +#' Default: "FAIL". +#' @param proxy an object of class `request` from a call to #' [crul::proxy()] #' @details #' [ckanr_setup()] sets CKAN connection details. ckanr's functions @@ -106,16 +113,18 @@ print.ckanr_settings <- function(x, ...) { #' ckanr_setup(url = "https://data.ontario.ca/", key = "some-CKAN-API-key") #' #' # ckanR developers/testers can run: -#' ckanr_setup(url = "https://data.ontario.ca/", key = "some-CKAN-API-key", -#' test_url = "http://test-ckan.gov/",test_key = "test-ckan-API-key", -#' test_did = "test-ckan-dataset-id",test_rid = "test-ckan-resource-id", -#' test_gid = "test-group-name", test_oid = "test-organzation-name", -#' test_behaviour = "FAIL") +#' ckanr_setup( +#' url = "https://data.ontario.ca/", key = "some-CKAN-API-key", +#' test_url = "http://test-ckan.gov/", test_key = "test-ckan-API-key", +#' test_did = "test-ckan-dataset-id", test_rid = "test-ckan-resource-id", +#' test_gid = "test-group-name", test_oid = "test-organization-name", +#' test_behaviour = "FAIL" +#' ) #' #' # Not specifying the default CKAN URL will reset the CKAN URL to its default #' # "https://data.ontario.ca/": #' ckanr_setup() -#' +#' #' # set a proxy #' ckanr_setup(proxy = crul::proxy("64.251.21.73:8080")) #' ckanr_settings() @@ -123,17 +132,16 @@ print.ckanr_settings <- function(x, ...) { #' ckanr_setup() #' ckanr_settings() ckanr_setup <- function( - url = "https://data.ontario.ca/", - key = NULL, - test_url = NULL, - test_key = NULL, - test_did = NULL, - test_rid = NULL, - test_gid = NULL, - test_oid = NULL, - test_behaviour = NULL, - proxy = NULL) { - + url = "https://data.ontario.ca/", + key = NULL, + test_url = NULL, + test_key = NULL, + test_did = NULL, + test_rid = NULL, + test_gid = NULL, + test_oid = NULL, + test_behaviour = NULL, + proxy = NULL) { Sys.setenv("CKANR_DEFAULT_URL" = url) if (!is.null(key)) Sys.setenv("CKANR_DEFAULT_KEY" = key) if (!is.null(test_url)) Sys.setenv("CKANR_TEST_URL" = test_url) @@ -151,36 +159,54 @@ ckanr_setup <- function( # #' @export #' @rdname ckanr_settings -get_default_url <- function(){ Sys.getenv("CKANR_DEFAULT_URL") } +get_default_url <- function() { + Sys.getenv("CKANR_DEFAULT_URL") +} #' @export #' @rdname ckanr_settings -get_default_key <- function(){ Sys.getenv("CKANR_DEFAULT_KEY") } +get_default_key <- function() { + Sys.getenv("CKANR_DEFAULT_KEY") +} #' @export #' @rdname ckanr_settings -get_test_url <- function(){ Sys.getenv("CKANR_TEST_URL") } +get_test_url <- function() { + Sys.getenv("CKANR_TEST_URL", unset = "http://localhost:5000") +} #' @export #' @rdname ckanr_settings -get_test_key <- function(){ Sys.getenv("CKANR_TEST_KEY") } +get_test_key <- function() { + Sys.getenv("CKANR_TEST_KEY") +} #' @export #' @rdname ckanr_settings -get_test_did <- function(){ Sys.getenv("CKANR_TEST_DID") } +get_test_did <- function() { + Sys.getenv("CKANR_TEST_DID") +} #' @export #' @rdname ckanr_settings -get_test_rid <- function(){ Sys.getenv("CKANR_TEST_RID") } +get_test_rid <- function() { + Sys.getenv("CKANR_TEST_RID") +} #' @export #' @rdname ckanr_settings -get_test_gid <- function(){ Sys.getenv("CKANR_TEST_GID") } +get_test_gid <- function() { + Sys.getenv("CKANR_TEST_GID") +} #' @export #' @rdname ckanr_settings -get_test_oid <- function(){ Sys.getenv("CKANR_TEST_OID") } +get_test_oid <- function() { + Sys.getenv("CKANR_TEST_OID") +} #' @export #' @rdname ckanr_settings -get_test_behaviour <- function(){ Sys.getenv("CKANR_TEST_BEHAVIOUR") } +get_test_behaviour <- function() { + Sys.getenv("CKANR_TEST_BEHAVIOUR", unset = "FAIL") +} diff --git a/man/ckanr_setup.Rd b/man/ckanr_setup.Rd index f077b0c..da8c0b0 100644 --- a/man/ckanr_setup.Rd +++ b/man/ckanr_setup.Rd @@ -22,7 +22,12 @@ ckanr_setup( \item{key}{A CKAN API key (optional, character)} -\item{test_url}{(optional, character) A valid CKAN URL for testing purposes} +\item{test_url}{(optional, character) A valid CKAN URL for testing purposes. +Default: "http://localhost:5000". +The default works with a local installation of CKAN via docker compose +running on port 5000, as well as for CI on GitHub. +This enables contributors to run the CI test suite on GitHub when submitting +a pull request.} \item{test_key}{(optional, character) A valid CKAN API key privileged to create datasets at \code{test_url}} @@ -39,7 +44,8 @@ create datasets at \code{test_url}} \code{test_url}} \item{test_behaviour}{(optional, character) Whether to fail ("FAIL") or skip -("SKIP") writing tests in case of problems with the configured test CKAN.} +("SKIP") writing tests in case of problems with the configured test CKAN. +Default: "FAIL".} \item{proxy}{an object of class \code{request} from a call to \code{\link[crul:proxies]{crul::proxy()}}} @@ -75,11 +81,13 @@ ckanr_setup(url = "https://data.ontario.ca/") ckanr_setup(url = "https://data.ontario.ca/", key = "some-CKAN-API-key") # ckanR developers/testers can run: -ckanr_setup(url = "https://data.ontario.ca/", key = "some-CKAN-API-key", - test_url = "http://test-ckan.gov/",test_key = "test-ckan-API-key", - test_did = "test-ckan-dataset-id",test_rid = "test-ckan-resource-id", - test_gid = "test-group-name", test_oid = "test-organzation-name", - test_behaviour = "FAIL") +ckanr_setup( + url = "https://data.ontario.ca/", key = "some-CKAN-API-key", + test_url = "http://test-ckan.gov/", test_key = "test-ckan-API-key", + test_did = "test-ckan-dataset-id", test_rid = "test-ckan-resource-id", + test_gid = "test-group-name", test_oid = "test-organization-name", + test_behaviour = "FAIL" +) # Not specifying the default CKAN URL will reset the CKAN URL to its default # "https://data.ontario.ca/": diff --git a/tests/testthat/test-changes.R b/tests/testthat/test-changes.R index dd923f1..82a455d 100644 --- a/tests/testthat/test-changes.R +++ b/tests/testthat/test-changes.R @@ -1,33 +1,33 @@ -context("changes") - -skip_on_cran() - -u <- get_test_url() -check_ckan(u) - -test_that("changes gives back expected class types", { - cat(u, sep = "\n") - a <- changes(url = u) - - expect_is(a, "list") - expect_is(a[[1]], "list") - expect_is(a[[1]]$user_id, "character") - expect_is(a[[1]]$data, "list") -}) - -test_that("changes works giving back json output", { - b <- changes(url = u, as = 'json') - b_df <- jsonlite::fromJSON(b) - - expect_is(b, "character") - expect_is(b_df, "list") - expect_is(b_df$result, "data.frame") -}) - -test_that("changes fails correctly", { - - expect_error(changes("adf"), "offset Invalid integer") - expect_error(changes(limit = "Adf"), "limit Invalid integer") - expect_error(changes("adf", url = "http://www.google.com"), - regexp = "404") -}) +# context("changes") +# +# skip_on_cran() +# +# u <- get_test_url() +# check_ckan(u) +# +# test_that("changes gives back expected class types", { +# cat(u, sep = "\n") +# a <- changes(url = u) +# +# expect_is(a, "list") +# expect_is(a[[1]], "list") +# expect_is(a[[1]]$user_id, "character") +# expect_is(a[[1]]$data, "list") +# }) +# +# test_that("changes works giving back json output", { +# b <- changes(url = u, as = 'json') +# b_df <- jsonlite::fromJSON(b) +# +# expect_is(b, "character") +# expect_is(b_df, "list") +# expect_is(b_df$result, "data.frame") +# }) +# +# test_that("changes fails correctly", { +# +# expect_error(changes("adf"), "offset Invalid integer") +# expect_error(changes(limit = "Adf"), "limit Invalid integer") +# expect_error(changes("adf", url = "http://www.google.com"), +# regexp = "404") +# }) diff --git a/tests/testthat/test-ckan_fetch.R b/tests/testthat/test-ckan_fetch.R index 7ecaec2..59c717e 100644 --- a/tests/testthat/test-ckan_fetch.R +++ b/tests/testthat/test-ckan_fetch.R @@ -2,6 +2,7 @@ context("ckan_fetch") skip_on_cran() +url <- get_test_url() rid <- get_test_rid() test_that("ckan_fetch returns error when file format can't be determined from URL", { @@ -17,7 +18,11 @@ check_ckan(u) test_that("ckan_fetch doesn't write any files to working directory when session = TRUE", { expect_identical(list.files(test_path()), { res <- resource_show(id = rid, as = "table") - df <- ckan_fetch(res$url) + # When running CKAN via docker compose, it returns its hostname as + # http://ckan:5000, but we can only reach it via localhost unless we patch + # /etc/hosts + url <- gsub("ckan", "localhost", res$url) + df <- ckan_fetch(url) list.files(test_path()) }) }) @@ -26,7 +31,8 @@ test_that("ckan_fetch doesn't retain any files in temporary directory when sessi dir <- tempdir() expect_identical(list.files(dir), { res <- resource_show(id = rid, as = "table") - df <- ckan_fetch(res$url) + url <- gsub("ckan", "localhost", res$url) + df <- ckan_fetch(url) list.files(dir) }) }) diff --git a/tests/testthat/test-organization_show.R b/tests/testthat/test-organization_show.R index b943981..ade02a6 100644 --- a/tests/testthat/test-organization_show.R +++ b/tests/testthat/test-organization_show.R @@ -28,9 +28,9 @@ test_that("organization_show gives back expected class types", { a <- organization_show(o, url=u) expect_is(a, "ckan_organization") - expect_is(a[[1]], "list") - expect_is(a[[1]][[1]]$name, "character") - expect_equal(as.integer(length(a)), 19L) + expect_is(a[[1]], "character") + expect_is(a$users[[1]]$name, "character") + expect_equal(as.integer(length(a)), 18L) a <- organization_show(o, url=u, include_datasets = TRUE) expect_equal(as.integer(a$package_count), dataset_num) @@ -45,5 +45,5 @@ test_that("organization_show works giving back json output", { expect_is(b, "character") expect_is(b_df, "list") expect_is(b_df$result, "list") - expect_equal(as.integer(length(b_df$result)), 19L) + expect_equal(as.integer(length(b_df$result)), 18L) }) diff --git a/tests/testthat/test-package_activity_list.R b/tests/testthat/test-package_activity_list.R index 1e06d13..5c3c478 100644 --- a/tests/testthat/test-package_activity_list.R +++ b/tests/testthat/test-package_activity_list.R @@ -1,44 +1,45 @@ -context("package_activity_list") - -skip_on_cran() -skip_on_ci() - -u <- get_test_url() -check_ckan(u) - -id <- package_list(limit = 1, url=u)[[1]] - -package_activity_num <- local({ - res <- crul::HttpClient$new(file.path(u, "dataset/activity", id))$get() - res$raise_for_status() - txt <- res$parse("UTF-8") - length(xml2::xml_find_all(xml2::read_html(txt), - '//ul[@data-module="activity-stream"]/li')) -}) - -test_that("package_activity_list gives back expected class types", { - - a <- package_activity_list(id, url=u, limit=30) - expect_is(a, "list") - expect_lt(length(a), 30 + 1) - a <- package_activity_list(id, url=u, limit=NULL) - expect_is(a, "list") - expect_equal(length(a), package_activity_num) -}) - -test_that("package_activity_list works giving back json output", { - b <- package_activity_list(id, url=u, as='json', limit=30) - b_df <- jsonlite::fromJSON(b) - expect_is(b, "character") - expect_is(b_df, "list") - expect_is(b_df$result, "data.frame") - expect_lt(nrow(b_df$result), 30 + 1) - - b <- package_activity_list(id, url=u, as='json', limit=NULL) - b_df <- jsonlite::fromJSON(b) - expect_is(b, "character") - expect_is(b_df, "list") - expect_is(b_df$result, "data.frame") - expect_equal(nrow(b_df$result), package_activity_num) -}) +# context("package_activity_list") +# +# skip_on_cran() +# skip_on_ci() +# +# u <- get_test_url() +# check_ckan(u) +# +# id <- package_list(limit = 1, url=u)[[1]] + +# TODO: create activity to show activity stream on dataset page +# package_activity_num <- local({ +# res <- crul::HttpClient$new(file.path(u, "dataset/activity", id))$get() +# res$raise_for_status() +# txt <- res$parse("UTF-8") +# length(xml2::xml_find_all(xml2::read_html(txt), +# '//ul[@data-module="activity-stream"]/li')) +# }) + +# test_that("package_activity_list gives back expected class types", { +# +# a <- package_activity_list(id, url=u, limit=30) +# expect_is(a, "list") +# expect_lt(length(a), 30 + 1) +# a <- package_activity_list(id, url=u, limit=NULL) +# expect_is(a, "list") +# # expect_equal(length(a), package_activity_num) +# }) +# +# test_that("package_activity_list works giving back json output", { +# b <- package_activity_list(id, url=u, as='json', limit=30) +# b_df <- jsonlite::fromJSON(b) +# expect_is(b, "character") +# expect_is(b_df, "list") +# expect_is(b_df$result, "data.frame") +# expect_lt(nrow(b_df$result), 30 + 1) +# +# b <- package_activity_list(id, url=u, as='json', limit=NULL) +# b_df <- jsonlite::fromJSON(b) +# expect_is(b, "character") +# expect_is(b_df, "list") +# expect_is(b_df$result, "data.frame") +# # expect_equal(nrow(b_df$result), package_activity_num) +# }) diff --git a/tests/testthat/test-tag_show.R b/tests/testthat/test-tag_show.R index 3c89178..ff08ed3 100644 --- a/tests/testthat/test-tag_show.R +++ b/tests/testthat/test-tag_show.R @@ -21,8 +21,8 @@ test_that("tag_show gives back expected class types", { a <- tag_show(t$name, include_datasets = TRUE, url=u) expect_is(a, "ckan_tag") - expect_is(a[[2]], "list") - #expect_equal(length(a[[2]]), tag_test_num) + expect_is(a$packages, "list") + #expect_equal(length(a$packages), tag_test_num) }) test_that("tag_show works giving back json output", { From 01b50021e66044c5b7676bbb3119bc8191a25cc9 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Wed, 29 May 2024 14:57:12 +0800 Subject: [PATCH 4/4] Fix dplyr.R * Update functions to match new DBI connection * Some function generics still need to be implemented * Tests are passing, but not covering much of the dplyr/DBI interface * Improve docstrings and examples --- NAMESPACE | 9 ++- R/dbi.R | 2 +- R/dplyr.R | 113 ++++++++++++++++++--------- man/dbplyr_edition.CKANConnection.Rd | 3 + man/dbplyr_edition.src_ckan.Rd | 15 ---- man/src_ckan.Rd | 31 ++++++-- man/tbl.CKANConnection.Rd | 22 ++++++ man/tbl.src_ckan.Rd | 22 ++++++ 8 files changed, 150 insertions(+), 67 deletions(-) delete mode 100644 man/dbplyr_edition.src_ckan.Rd create mode 100644 man/tbl.CKANConnection.Rd create mode 100644 man/tbl.src_ckan.Rd diff --git a/NAMESPACE b/NAMESPACE index 23fb8f6..08dcced 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -22,12 +22,12 @@ S3method(as.ckan_user,character) S3method(as.ckan_user,ckan_user) S3method(as.ckan_user,list) S3method(db_begin,CKANConnection) +S3method(db_desc,CKANConnection) S3method(db_has_table,CKANConnection) S3method(db_insert_into,CKANConnection) S3method(db_query_rows,CKANConnection) S3method(dbplyr_edition,CKANConnection) -S3method(dbplyr_edition,src_ckan) -S3method(format,src_ckan) +S3method(format,CKANConnection) S3method(print,ckan_group) S3method(print,ckan_organization) S3method(print,ckan_package) @@ -40,7 +40,8 @@ S3method(sql_query_explain,CKANConnection) S3method(sql_query_fields,CKANConnection) S3method(sql_translation,CKANConnection) S3method(sql_translation,src_ckan) -S3method(src_tbls,src_ckan) +S3method(src_tbls,CKANConnection) +S3method(tbl,CKANConnection) S3method(tbl,src_ckan) export("%>%") export(as.ckan_group) @@ -136,6 +137,7 @@ importFrom(dbplyr,base_scalar) importFrom(dbplyr,base_win) importFrom(dbplyr,build_sql) importFrom(dbplyr,dbplyr_edition) +importFrom(dbplyr,sql) importFrom(dbplyr,sql_prefix) importFrom(dbplyr,sql_query_explain) importFrom(dbplyr,sql_query_fields) @@ -152,7 +154,6 @@ importFrom(dplyr,db_has_table) importFrom(dplyr,db_insert_into) importFrom(dplyr,db_list_tables) importFrom(dplyr,db_query_rows) -importFrom(dplyr,sql) importFrom(dplyr,src_tbls) importFrom(dplyr,tbl) importFrom(jsonlite,fromJSON) diff --git a/R/dbi.R b/R/dbi.R index 7f29eff..b9f3c78 100644 --- a/R/dbi.R +++ b/R/dbi.R @@ -131,7 +131,7 @@ setMethod("dbGetException", "CKANConnection", setMethod("dbGetInfo", "CKANConnection", def = function(dbObj, ...) { - cat(sprintf("url: %s\n", dbObj@url)) + cat(sprintf("CKAN URL: %s\n", dbObj@url)) } ) diff --git a/R/dplyr.R b/R/dplyr.R index 60d33ed..b74efc6 100644 --- a/R/dplyr.R +++ b/R/dplyr.R @@ -2,23 +2,35 @@ #' #' Use `src_ckan` to connect to an existing CKAN instance and `tbl` to #' connect to tables within that CKAN based on the DataStore Data API. +#' +#' The function returns a +#' [`DBI::dbConnect()`](https://dbi.r-dbi.org/reference/DBIConnection-class.html) +#' object, which can be then used by +#' [dbplyr](https://dbplyr.tidyverse.org/articles/dbplyr.html) and +#' #' #' @param url, the url of the CKAN instance #' @examples \dontrun{ -#' library("dplyr") -#' #' # To connect to a CKAN instance first create a src: -#' my_ckan <- src_ckan("http://demo.ckan.org") -#' -#' # List all tables in the CKAN instance -#' db_list_tables(my_ckan$con) +#' my_ckan <- src_ckan("https://www.data.qld.gov.au/") +#' +#' summary(my_ckan) +#' dplyr::db_desc(my_ckan) +#' +#' # dplyr has changed its dbplyr backend API to version 2 +#' dbplyr::dbplyr_edition(my_ckan) #' -#' # Then reference a tbl within that src -#' my_tbl <- tbl(src = my_ckan, name = "44d7de5f-7029-4f3a-a812-d7a70895da7d") +#' # List all tables in the CKAN database +#' tbl_list <- DBI::dbListTables(my_ckan, limit=5) +#' tbl_list #' -#' # You can use the dplyr verbs with my_tbl. For example: -#' dplyr::filter(my_tbl, GABARITO == "C") +#' # Create an in-memory table from a CKAN database table +#' tbl1 <- dplyr::tbl(my_ckan, tbl_list[1]) +#' tbl2 <- dplyr::tbl(my_ckan, "587f65ae-6675-4b8e-bac5-606ce7f4446a") #' +#' # You can use the dplyr verbs with tbl. For example: +#' dplyr::filter(tbl1, variable_name == "value") +#' dplyr::select(tbl2, `Density class`, Full) #' } #' @aliases dplyr-interface #' @export @@ -26,42 +38,71 @@ src_ckan <- function(url) { if (!requireNamespace("dplyr", quietly = TRUE)) { stop("Please install dplyr", call. = FALSE) } + drv <- new("CKANDriver") - con <- dbConnect(drv, url = url) - info <- dbGetInfo(con) - src_sql("ckan", con, info = info) + DBI::dbConnect(drv, url = url) } -#'@export -#'@importFrom dplyr tbl -tbl.src_ckan <- function(src, from, ..., name = NULL) { - if (is.null(name)) { - tbl_sql("ckan", src = src, from = sql(from), ...) - } else { - tbl_sql(subclass = "ckan", - src = src, - from = sql(sprintf('SELECT * FROM "%s"', name))) - } +#' An implementation of `dplyr::tbl()` for `ckanr` +#' +#' This implementation converts the `DBI::dbConnect()` connection returned by +#' `ckanr::src_ckan()` into a structure that nests the connection under a key +#' `con`. This is expected by `dbplyr::tbl_sql()`. +#' +#' @param src A `DBI::dbConnect()` database connection. +#' @param name The table name which will be used as the SQL FROM clause. +#' This is vulnerable to SQL injections, so use only with known safe names, +#' e.g. CKAN resource IDs. +#' @param ... Extra arguments will be passed on to `dbplyr::tbl_sql()`. +#' @export +#' @importFrom dplyr tbl +#' @importFrom dbplyr tbl_sql sql +tbl.src_ckan <- function(src, name, ...) { + dplyr_src = structure(list(con=src)) + dbplyr::tbl_sql( + subclass = "CKANConnection", + src = dplyr_src, + from = dbplyr::sql(sprintf('SELECT * FROM "%s"', name)), + ... + ) +} + +#' An implementation of `dplyr::tbl()` for `ckanr` +#' +#' This implementation converts the `DBI::dbConnect()` connection returned by +#' `ckanr::src_ckan()` into a structure that nests the connection under a key +#' `con`. This is expected by `dbplyr::tbl_sql()`. +#' +#' @param src A dplyr data source +#' @param name The table name. If given, `from` will be ignored, and the name +#' will be used as string. This is vulnerable to SQL injections, so use only +#' with known safe names, e.g. CKAN resource IDs. +#' @param ... Extra arguments will be passed on to `dbplyr::tbl_sql()`. +#' @export +#' @importFrom dplyr tbl +#' @importFrom dbplyr tbl_sql sql +tbl.CKANConnection <- function(src, name, ...) { + tbl.src_ckan(src, name, ...) } +#' @export #' @importFrom dplyr db_desc -db_desc.src_ckan <- function(x) { - info <- x$info - sprintf("ckan url: %s", x$con@url) +db_desc.CKANConnection <- function(x) { + sprintf("CKAN URL: %s", x@url) } #' @export #' @importFrom dplyr src_tbls -src_tbls.src_ckan <- function(x, ..., limit = 6) { +src_tbls.CKANConnection <- function(x, ..., limit = 6) { if (!is.null(limit)) { - c(dbListTables(x$con, limit = limit)) + c(dbListTables(x, limit = limit)) } else { - dbListTables(x$con) + dbListTables(x) } } #' @export -format.src_ckan <- function(x, ...) { +format.CKANConnection <- function(x, ...) { .metadata <- ds_search("_table_metadata", url = x$con@url, limit = 6) x1 <- sprintf("%s", db_desc(x)) x2 <- sprintf("total tbls: %d", .metadata$total) @@ -157,19 +198,13 @@ db_query_rows.CKANConnection <- function(con, sql, ...) { #' Use dbplyr 2.0.0 generics #' See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition #' Introduced in 0.8.0 +#' @param con A database connection. #' @importFrom dbplyr dbplyr_edition #' @export dbplyr_edition.CKANConnection <- function(con) 2L -#' Use dbplyr 2.0.0 generics -#' See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition -#' Introduced in 0.8.0 -#' @importFrom dbplyr dbplyr_edition -#' @export -dbplyr_edition.src_ckan <- function(con) 2L - -#' @importFrom dplyr db_list_tables sql -#' @importFrom dbplyr base_agg base_scalar base_win build_sql sql_prefix +#' @importFrom dplyr db_list_tables +#' @importFrom dbplyr base_agg base_scalar base_win build_sql sql sql_prefix #' sql_translator sql_variant src_sql tbl_sql #' sql_query_fields sql_query_select sql_query_wrap sql_query_explain sql_translation NULL diff --git a/man/dbplyr_edition.CKANConnection.Rd b/man/dbplyr_edition.CKANConnection.Rd index 0bf2917..69835c9 100644 --- a/man/dbplyr_edition.CKANConnection.Rd +++ b/man/dbplyr_edition.CKANConnection.Rd @@ -8,6 +8,9 @@ Introduced in 0.8.0} \usage{ \method{dbplyr_edition}{CKANConnection}(con) } +\arguments{ +\item{con}{A database connection.} +} \description{ Use dbplyr 2.0.0 generics See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition diff --git a/man/dbplyr_edition.src_ckan.Rd b/man/dbplyr_edition.src_ckan.Rd deleted file mode 100644 index adc8873..0000000 --- a/man/dbplyr_edition.src_ckan.Rd +++ /dev/null @@ -1,15 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/dplyr.R -\name{dbplyr_edition.src_ckan} -\alias{dbplyr_edition.src_ckan} -\title{Use dbplyr 2.0.0 generics -See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition -Introduced in 0.8.0} -\usage{ -\method{dbplyr_edition}{src_ckan}(con) -} -\description{ -Use dbplyr 2.0.0 generics -See https://dbplyr.tidyverse.org/articles/backend-2.html#nd-edition -Introduced in 0.8.0 -} diff --git a/man/src_ckan.Rd b/man/src_ckan.Rd index 5a8b089..4dcc287 100644 --- a/man/src_ckan.Rd +++ b/man/src_ckan.Rd @@ -14,21 +14,36 @@ src_ckan(url) Use \code{src_ckan} to connect to an existing CKAN instance and \code{tbl} to connect to tables within that CKAN based on the DataStore Data API. } +\details{ +The function returns a +\href{https://dbi.r-dbi.org/reference/DBIConnection-class.html}{\code{DBI::dbConnect()}} +object, which can be then used by +\href{https://dbplyr.tidyverse.org/articles/dbplyr.html}{dbplyr} and +} \examples{ \dontrun{ -library("dplyr") +library(dplyr) +library(DBI) # To connect to a CKAN instance first create a src: -my_ckan <- src_ckan("http://demo.ckan.org") +my_ckan <- src_ckan("https://www.data.qld.gov.au/") + +summary(my_ckan) +dplyr::db_desc(my_ckan) -# List all tables in the CKAN instance -db_list_tables(my_ckan$con) +# dplyr has changed its dbplyr backend API to version 2 +dbplyr::dbplyr_edition(my_ckan) -# Then reference a tbl within that src -my_tbl <- tbl(src = my_ckan, name = "44d7de5f-7029-4f3a-a812-d7a70895da7d") +# List all tables in the CKAN database +tbl_list <- DBI::dbListTables(my_ckan, limit=5) +tbl_list -# You can use the dplyr verbs with my_tbl. For example: -dplyr::filter(my_tbl, GABARITO == "C") +# Create an in-memory table from a CKAN database table +tbl1 <- dplyr::tbl(my_ckan, tbl_list[1]) +tbl2 <- dplyr::tbl(my_ckan, "587f65ae-6675-4b8e-bac5-606ce7f4446a") +# You can use the dplyr verbs with tbl. For example: +dplyr::filter(tbl1, variable_name == "value") +dplyr::select(tbl2, `Density class`, Full) } } diff --git a/man/tbl.CKANConnection.Rd b/man/tbl.CKANConnection.Rd new file mode 100644 index 0000000..dd1118e --- /dev/null +++ b/man/tbl.CKANConnection.Rd @@ -0,0 +1,22 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/dplyr.R +\name{tbl.CKANConnection} +\alias{tbl.CKANConnection} +\title{An implementation of \code{dplyr::tbl()} for \code{ckanr}} +\usage{ +\method{tbl}{CKANConnection}(src, name, ...) +} +\arguments{ +\item{src}{A dplyr data source} + +\item{name}{The table name. If given, \code{from} will be ignored, and the name +will be used as string. This is vulnerable to SQL injections, so use only +with known safe names, e.g. CKAN resource IDs.} + +\item{...}{Extra arguments will be passed on to \code{dbplyr::tbl_sql()}.} +} +\description{ +This implementation converts the \code{DBI::dbConnect()} connection returned by +\code{ckanr::src_ckan()} into a structure that nests the connection under a key +\code{con}. This is expected by \code{dbplyr::tbl_sql()}. +} diff --git a/man/tbl.src_ckan.Rd b/man/tbl.src_ckan.Rd new file mode 100644 index 0000000..d7b5ee4 --- /dev/null +++ b/man/tbl.src_ckan.Rd @@ -0,0 +1,22 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/dplyr.R +\name{tbl.src_ckan} +\alias{tbl.src_ckan} +\title{An implementation of \code{dplyr::tbl()} for \code{ckanr}} +\usage{ +\method{tbl}{src_ckan}(src, name, ...) +} +\arguments{ +\item{src}{A \code{DBI::dbConnect()} database connection.} + +\item{name}{The table name which will be used as the SQL FROM clause. +This is vulnerable to SQL injections, so use only with known safe names, +e.g. CKAN resource IDs.} + +\item{...}{Extra arguments will be passed on to \code{dbplyr::tbl_sql()}.} +} +\description{ +This implementation converts the \code{DBI::dbConnect()} connection returned by +\code{ckanr::src_ckan()} into a structure that nests the connection under a key +\code{con}. This is expected by \code{dbplyr::tbl_sql()}. +}