Merge remote-tracking branch 'origin/main' into mthomson/row_number_s…

…nowflake

DESCRIPTION

@@ -17,39 +17,39 @@ License: MIT + file LICENSE
 URL: https://dbplyr.tidyverse.org/, https://github.com/tidyverse/dbplyr
 BugReports: https://github.com/tidyverse/dbplyr/issues
 Depends: 
-    R (>= 3.1)
+    R (>= 3.6)
 Imports: 
     blob (>= 1.2.0),
-    cli (>= 3.4.1),
-    DBI (>= 1.0.0),
-    dplyr (>= 1.1.0),
-    glue (>= 1.2.0),
+    cli (>= 3.6.1),
+    DBI (>= 1.1.3),
+    dplyr (>= 1.1.2),
+    glue (>= 1.6.2),
     lifecycle (>= 1.0.3),
     magrittr,
     methods,
-    pillar (>= 1.5.0),
+    pillar (>= 1.9.0),
     purrr (>= 1.0.1),
     R6 (>= 2.2.2),
-    rlang (>= 1.1.0),
-    tibble (>= 1.4.2),
+    rlang (>= 1.1.1),
+    tibble (>= 3.2.1),
     tidyr (>= 1.3.0),
     tidyselect (>= 1.2.0),
     utils,
-    vctrs (>= 0.5.2),
-    withr
+    vctrs (>= 0.6.3),
+    withr (>= 2.5.0)
 Suggests:
     bit64,
     covr,
     knitr,
     Lahman,
     nycflights13,
     odbc,
-    RMariaDB (>= 1.0.2),
+    RMariaDB (>= 1.2.2),
     rmarkdown,
-    RPostgres (>= 1.1.3),
+    RPostgres (>= 1.4.5),
     RPostgreSQL,
-    RSQLite (>= 2.2.15),
-    testthat (>= 3.0.2)
+    RSQLite (>= 2.3.1),
+    testthat (>= 3.1.10)
 VignetteBuilder: 
     knitr
 Config/Needs/website: tidyverse/tidytemplate

NAMESPACE

@@ -30,6 +30,13 @@ S3method(copy_to,src_sql)
 S3method(count,tbl_lazy)
 S3method(cross_join,tbl_lazy)
 S3method(db_analyze,DBIConnection)
+S3method(db_col_types,DBIConnection)
+S3method(db_col_types,MariaDBConnection)
+S3method(db_col_types,MySQL)
+S3method(db_col_types,MySQLConnection)
+S3method(db_col_types,PostgreSQL)
+S3method(db_col_types,PqConnection)
+S3method(db_col_types,TestConnection)
 S3method(db_collect,DBIConnection)
 S3method(db_compute,DBIConnection)
 S3method(db_connection_describe,DBIConnection)
@@ -135,10 +142,6 @@ S3method(format,ident)
 S3method(format,sql)
 S3method(format,src_sql)
 S3method(full_join,tbl_lazy)
-S3method(get_col_types,DBIConnection)
-S3method(get_col_types,MariaDBConnection)
-S3method(get_col_types,PqConnection)
-S3method(get_col_types,TestConnection)
 S3method(group_by,tbl_lazy)
 S3method(group_size,tbl_sql)
 S3method(group_vars,tbl_lazy)
@@ -429,6 +432,7 @@ export(build_sql)
 export(copy_inline)
 export(copy_lahman)
 export(copy_nycflights13)
+export(db_col_types)
 export(db_collect)
 export(db_compute)
 export(db_connection_describe)

NEWS.md

@@ -1,5 +1,21 @@
 # dbplyr (development version)
 
+* Joins
+
+  * `*_join()` after `full_join()` works again (@mgirlich, #1178).
+
+  * `*_join()` now allows specifying the relationship argument. It must be
+    `NULL` or `"many-to-many"` (@bairdj, #1305).
+
+  * Queries now qualify `*` with the table alias for better compatibility (@mgirlich, #1003).
+
+  * `full_join()` can now handle column names that only differ in case (@ejneer, #1255).
+
+  * The `na_matches` argument of `semi_join()` and `anti_join()` works again
+    (@mgirlich, #1211).
+
+  * Joins now work again for Pool and Oracle connections (@mgirlich, #1177, #1181).
+
 * `dbplyr_pivot_wider_spec()` is now exported. Unlike `pivot_wider()` this can
   be lazy. Note that this will be removed soon after `pivot_wider_spec()`
   becomes a generic (@mgirlich).
@@ -79,82 +95,141 @@
 
 * The `cte` argument of `show_query()` and `remote_query()` is deprecated (@mgirlich, #1146).
 
-* `any()` and `all()` now work for MS SQL (@ejneer, #1273).
+## Breaking changes
 
-* Fixed negation of bit (boolean) fields in MS SQL (@ejneer, #1239)
+* `remote_name()` now returns a string with the name of the table. To get the
+  qualified identifier use the newly added `remote_table()` (@mgirlich, #1280).
 
-* A sequence of `union()` resp. `union_all()` now produces a flat query
-  instead of subqueries (@mgirlich, #1269).
+* The functions `simulate_vars()` and `simulate_vars_is_typed()` were removed
+  as they weren't used anymore and tidyselect now offers `tidyselect_data_proxy()`
+  and `tidyselect_data_has_predicates()` (@mgirllich, #1199).
 
-* Using `compute(temporary = FALSE)` without providing a name is now
-  deprecated (@mgirlich, #1154).
+* `translate_sql()` now requires the `con` argument (@mgirlich, #1311).
 
-* Added translation for `nzchar()` (@MichaelChirico, @mgirlich, #1094).
+* `sql_query_append()`, `sql_query_insert()`, `sql_query_update()`,
+  `sql_query_upsert()`, and `sql_query_delete()` changed their arguments to
+  make them more consistent to the other `sql_query_*()` functions:
+
+  * `x_name` was renamed to `table`.
+  * `y` was renamed to `from` and must now be a table identifier or SQL instead
+    of a lazy table.
+  * `sql_query_append()` and `sql_query_insert()` have gained the argument `cols`.
 
-* MySQL/MariaDB:
-  * Fix translation of `as.numeric()`, `as.POSIXct()`, `as_datetime()`, and
-    `as.integer64()` (@avsdev-cw, #1189).
+* The first argument of `ntile()` has been renamed from `order_by` to `x` to
+  match the interface of `dplyr::ntile()` (@mgirlich, #1242).
 
-* `full_join()` can now handle column names that only differ in case (@ejneer, #1255).
+* Removed argument `src` of `tbl_lazy()` after it has been deprecated for years
+  (@mgirlich, #1208).
 
-* Subqueries now also get an alias for SQLite. This makes it consistent with
-  other backends and simplifies the implementation.
+* Using `compute(temporary = FALSE)` without providing a name is now
+  deprecated (@mgirlich, #1154).
 
-* The translation of `if_any()` and `if_all()` is now wrapped in parentheses.
-  This makes sure it can be combined via `&` with other conditions (@mgirlich, #1153).
+* The `vars` argument of `translate_sql()` has been removed after it threw an
+  error for the last 7 years (@mgirlich).
 
-* `pivot_wider()` now matches tidyr `NA` column handling (@ejneer #1238).
+* `sql_not_supported()` now expects a function name without parentheses.
 
-* Using a function with a namespace in `across()` now works, e.g.
-  `across(x, dplyr::dense_rank)` (@mgirlich, #1231).
+## Improved SQL
 
-* The dots in `db_copy_to()` are now passed to `db_write_table()` (@mgirlich, #1237).
+* A `semi/anti_join()` where `y` is filtered is now inlined when possible (@mgirlich, #884).
 
-* Can now use `select()` again after using `arrange(desc(x))` (@ejneer, #1240).
+* A sequence of `union()` resp. `union_all()` now produces a flat query
+  instead of subqueries (@mgirlich, #1269).
 
-* The first argument of `ntile()` has been renamed from `order_by` to `x` to
-  match the interface of `dplyr::ntile()` (@mgirlich, #1242).
+## Minor improvements and bug fixes
+* Teradata:
+  * `as.Date(x)` is now translate to `CAST(x AS DATE)` again unless `x` is a
+    string (@mgirlich, #1285).
+
+* Snowflake: 
+  * `na.rm = TRUE` is now respected in `pmin()` and `pmax()` instead of being silently ignored (@fh-mthomson, #1329)
 
-* Removed argument `src` of `tbl_lazy()` after it has been deprecated for years
-  (@mgirlich, #1208).
+* The `overwrite` argument of `db_copy_to()` now actually works.
 
-* `sql_join_suffix()` gains the argument `suffix` so that methods can check
-  whether the suffix is valid for the backend (@mgirlich).
+* `db_write_table()` and `db_save_query()` gain the `overwrite` argument.
 
-* `sql_query_append()`, `sql_query_insert()`, `sql_query_update()`,
-  `sql_query_upsert()`, and `sql_query_delete()` changed their arguments to
-  make them more consistent to the other `sql_query_*()` functions:
-
-  * `x_name` was renamed to `table`.
-  * `y` was renamed to `from` and must now be a table identifier or SQL instead
-    of a lazy table.
-  * `sql_query_append()` and `sql_query_insert()` have gained the argument `cols`.
+* The dots in `db_copy_to()` are now passed to `db_write_table()` (@mgirlich, #1237).
 
-* The `na_matches` argument of `semi_join()` and `anti_join()` works again
-  (@mgirlich, #1211).
+* Added translation for `str_detect()`, `str_starts()` and `str_ends()` with
+  fixed patterns (@mgirlich, #1009).
 
-* The `vars` argument of `translate_sql()` has been removed after it threw an
-  error for the last 7 years (@mgirlich).
+* Added translation for `nzchar()` (@MichaelChirico, @mgirlich, #1094).
 
 * Added translation for `runif()` (@mgirlich, #1200).
 
 * `sql_random()` is now deprecated. It was used to power `slice_sample()` which
   is now done via the translation for `runif()` (@mgirlich, #1200).
+
+* `nth()`, `first()`, and `last()` now support the `na_rm` argument (@mgirlich, #1193).
 
-* DuckDB now supports the `returning` argument of `rows_*()`.
+* The translation of `if_any()` and `if_all()` is now wrapped in parentheses.
+  This makes sure it can be combined via `&` with other conditions (@mgirlich, #1153).
 
-* `nth()`, `first()`, and `last()` now support the `na_rm` argument (@mgirlich, #1193).
+* `slice_min/max()` can now order by multiple variables like dplyr, e.g. use
+  `slice_min(lf, tibble(x, y))` (@mgirlich, #1167).
 
-* `distinct()` + `head()` now work for Teradata (@mgirlich, #685).
+* `slice_*()` now supports the data masking pronouns `.env` and `.data` (@mgirlich, #1294).
 
-* `copy_inline()` now works for MariaDB (@mgirlich, #1188).
+* Can now use `select()` again after using `arrange(desc(x))` (@ejneer, #1240).
 
-* `*_join()` after `full_join()` works again (@mgirlich, #1178).
+* The columns generated when using a window function in `filter()` are now named
+  `col01` etc. instead of `q01()` (@mgirlich, #1258).
 
-* The `rows_*()` functions now also work inside a transaction for Postgres
-  (@mgirlich, #1183).
+* `tbl()` now informs when the user probably forgot to wrap the table identifier
+  with `in_schema()` or `sql()` (@mgirlich, #1287).
+
+* `pivot_wider()` now matches tidyr `NA` column handling (@ejneer #1238).
 
-* Added translation for `!=` to `<>` for Microsoft Access (@erikvona, #1219).
+* Using a function with a namespace in `across()` now works, e.g.
+  `across(x, dplyr::dense_rank)` (@mgirlich, #1231).
+
+* `sql_join_suffix()` gains the argument `suffix` so that methods can check
+  whether the suffix is valid for the backend (@mgirlich).
+
+* Added `db_supports_table_alias_with_as()` to customise whether a backend supports
+  specifying a table alias with `AS` or not (@mgirlich).
+
+## Backend specific improvements
+
+* Access
+  * Added translation for `!=` to `<>` (@erikvona, #1219).
+
+* DuckDB
+  * now supports the `returning` argument of `rows_*()`.
+
+* MySQL/MariaDB:
+  * `rows_update()` and `rows_patch()` now give an informative error when the
+    unsupported `returning` argument is used (@mgirlich, #1279).
+  * `rows_upsert()` now gives an informative error that it isn't supported
+    (@mgirlich, #1279).
+  * `rows_*()` use the column types of `x` when auto copying `y` (@mgirlich, #1327).
+  * `copy_inline()` now works (@mgirlich, #1188).
+  * Fix translation of `as.numeric()`, `as.POSIXct()`, `as_datetime()`, and
+    `as.integer64()` (@avsdev-cw, #1189).
+
+* Oracle:
+  * Fix translation of `rows_upsert()` (@mgirlich, @TBlackmore, #1286)
+  * `head(n)` is now translated to `WHERE ROWNUM <= n` to also support old
+    versions <= 11.2 (@JeremyPasco, #1292).
+
+* Postgres
+  * The `rows_*()` functions now also work inside a transaction (@mgirlich, #1183).
+
+* SQLite
+  * Subqueries now also get an alias. This makes it consistent with other
+    backends and simplifies the implementation.
+
+* SQL Server
+  * `distinct(.keep_all = TRUE)` now works (@mgirlich, #1053).
+  * The translation of `between()` now also works when used in `mutate()`
+    (@mgirlich, #1241).
+  * `any()` and `all()` now work (@ejneer, #1273).
+  * Fixed negation of bit (boolean) fields (@ejneer, #1239)
+
+* Teradata
+  * `distinct()` + `head()` now work (@mgirlich, #685).
+  * `as.Date(x)` is now translate to `CAST(x AS DATE)` again unless `x` is a
+    string (@mgirlich, #1285).
 
 # dbplyr 2.3.3
 

R/backend-mysql.R

@@ -50,6 +50,17 @@ db_connection_describe.MySQL <- db_connection_describe.MariaDBConnection
 #' @export
 db_connection_describe.MySQLConnection <- db_connection_describe.MariaDBConnection
 
+#' @export
+db_col_types.MariaDBConnection <- function(con, table, call) {
+  table <- as_table_ident(table, error_call = call)
+  col_info_df <- DBI::dbGetQuery(con, glue_sql2(con, "SHOW COLUMNS FROM {.tbl table};"))
+  set_names(col_info_df[["Type"]], col_info_df[["Field"]])
+}
+#' @export
+db_col_types.MySQL <- db_col_types.MariaDBConnection
+#' @export
+db_col_types.MySQLConnection <- db_col_types.MariaDBConnection
+
 #' @export
 sql_translation.MariaDBConnection <- function(con) {
   sql_variant(

R/backend-postgres.R

@@ -404,4 +404,17 @@ db_supports_table_alias_with_as.PostgreSQL <- function(con) {
   TRUE
 }
 
+#' @export
+db_col_types.PqConnection <- function(con, table, call) {
+  table <- as_table_ident(table, error_call = call)
+  res <- DBI::dbSendQuery(con, glue_sql2(con, "SELECT * FROM {.tbl table} LIMIT 0"))
+  on.exit(DBI::dbClearResult(res))
+  DBI::dbFetch(res, n = 0)
+  col_info_df <- DBI::dbColumnInfo(res)
+  set_names(col_info_df[[".typname"]], col_info_df[["name"]])
+}
+
+#' @export
+db_col_types.PostgreSQL <- db_col_types.PqConnection
+
 utils::globalVariables(c("strpos", "%::%", "%FROM%", "%ILIKE%", "DATE", "EXTRACT", "TO_CHAR", "string_agg", "%~*%", "%~%", "MONTH", "DOY", "DATE_TRUNC", "INTERVAL", "FLOOR", "WEEK"))

R/db.R

@@ -7,6 +7,8 @@
 #' * `dbplyr_edition()` declares which version of the dbplyr API you want.
 #'    See below for more details.
 #'
+#' * `db_col_types()` returns the column types of a table.
+#'
 #' @section dbplyr 2.0.0:
 #' dbplyr 2.0.0 renamed a number of generics so that they could be cleanly moved
 #' from dplyr to dbplyr. If you have an existing backend, you'll need to rename
@@ -58,12 +60,14 @@ sql_join_suffix.DBIConnection <- function(con, suffix, ...) {
 db_sql_render <- function(con, sql, ..., cte = FALSE, sql_options = NULL) {
   check_bool(cte)
   if (cte) {
-    lifecycle::deprecate_soft(
+  lifecycle::deprecate_soft(
       when = "2.4.0",
       what = "db_sql_render(cte)",
       with = I("db_sql_render(sql_options = sql_options(cte = TRUE))")
     )
-    sql_options <- sql_options(cte = TRUE)
+    sql_options <- sql_options %||% sql_options(cte = TRUE)
+    out <- db_sql_render(con, sql, sql_options = sql_options)
+    return(out)
   }
 
   if (is.null(sql_options)) {
@@ -80,6 +84,32 @@ db_sql_render.DBIConnection <- function(con, sql, ..., cte = FALSE, sql_options
   sql_render(sql, con = con, ..., sql_options = sql_options)
 }
 
+#' @rdname db-misc
+#' @export
+db_col_types <- function(con, table, call) {
+  if (is_null(table)) {
+    return(NULL)
+  }
+
+  UseMethod("db_col_types")
+}
+
+#' @export
+db_col_types.TestConnection <- function(con, table, call) {
+  NULL
+}
+
+#' @export
+db_col_types.DBIConnection <- function(con, table, call) {
+  NULL
+}
+# add a default method so that packages that haven't implemented `db_col_types()`
+# keep working, e.g. {Pool}
+#' @export
+db_col_types.default <- function(con, table, call) {
+  NULL
+}
+
 #' Options for generating SQL
 #'
 #' @param cte If `FALSE`, the default, subqueries are used. If `TRUE` common