From 572a9a7d3825d28dd29c8457509e866209c78394 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 17:08:36 +0000 Subject: [PATCH 01/27] chore: pre-commit autoupdate (#4703) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 775032649862..c269b4c13ef7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: # rustfmt handles rust files, and in some snapshots we expect trailing spaces. exclude: '.*\.(rs|snap)$' - repo: https://github.com/crate-ci/typos - rev: v1.22.9 + rev: v1.23.1 hooks: - id: typos # https://github.com/crate-ci/typos/issues/347 @@ -25,7 +25,7 @@ repos: # https://github.com/PRQL/prql/issues/3078 - prettier-plugin-go-template - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.0 + rev: v0.5.1 hooks: - id: ruff args: [--fix] From 18ab9760009d8f58fc38bccbe6f8223b1b4127cc Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Mon, 8 Jul 2024 10:10:41 -0700 Subject: [PATCH 02/27] internal: Assume chumsky parser outputs are `Debug` (#4700) --- prqlc/prqlc-parser/src/error/parse_error.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/prqlc/prqlc-parser/src/error/parse_error.rs b/prqlc/prqlc-parser/src/error/parse_error.rs index 94d92262b6d2..3d0efcf696db 100644 --- a/prqlc/prqlc-parser/src/error/parse_error.rs +++ b/prqlc/prqlc-parser/src/error/parse_error.rs @@ -1,6 +1,6 @@ use core::fmt; use std::collections::HashSet; -use std::fmt::Display; +use std::fmt::{Debug, Display}; use std::hash::Hash; use crate::error::{Error, ErrorSource, Reason, WithErrorInfo}; @@ -8,7 +8,7 @@ use crate::lexer::lr::TokenKind; use crate::span::Span; #[derive(Clone, Debug)] -pub struct ChumError { +pub struct ChumError { span: Span, reason: Option, expected: HashSet>, @@ -18,7 +18,7 @@ pub struct ChumError { pub type PError = ChumError; -impl ChumError { +impl ChumError { ///Create an error with a custom error message. pub fn custom(span: Span, msg: M) -> Self { Self { @@ -59,7 +59,7 @@ impl ChumError { /// /// This can be used to unify the errors between parsing stages that operate upon two forms of input (for example, /// the initial lexing stage and the parsing stage in most compilers). - pub fn map U>(self, mut f: F) -> ChumError { + pub fn map U>(self, mut f: F) -> ChumError { ChumError { span: self.span, reason: self.reason, @@ -70,7 +70,7 @@ impl ChumError { } } -impl chumsky::Error for ChumError { +impl chumsky::Error for ChumError { type Span = Span; type Label = &'static str; @@ -143,7 +143,7 @@ impl chumsky::Error for ChumError PartialEq for ChumError { +impl PartialEq for ChumError { fn eq(&self, other: &Self) -> bool { self.span == other.span && self.found == other.found @@ -152,9 +152,9 @@ impl PartialEq for ChumError { && self.label == other.label } } -impl Eq for ChumError {} +impl Eq for ChumError {} -impl fmt::Display for ChumError { +impl fmt::Display for ChumError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { // TODO: Take `self.reason` into account From c73c3fb8bacb190fa4aba9082dc836402d9736ff Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Mon, 8 Jul 2024 11:12:10 -0700 Subject: [PATCH 03/27] test: Disable a megalinter check (#4708) --- .mega-linter.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.mega-linter.yaml b/.mega-linter.yaml index ddfd1c779e4c..8557dfecf79b 100644 --- a/.mega-linter.yaml +++ b/.mega-linter.yaml @@ -41,5 +41,9 @@ DISABLE_ERRORS_LINTERS: # required in order to use ts-standard." was worth fixing, from #3608. Happy # for someone more informed to turn it back on. - TYPESCRIPT_STANDARD + # Disabled for now, as @max-sixty didn't know how to fix + # `./prqlc/bindings/php/tests/CompilerTest.php (trailing_comma_in_multiline)`. + # Fine for someone else to take a look. + - PHP_PHPCSFIXER PHP_PHPCS_ARGUMENTS: - --standard=PSR12 From ac710c2197c5ffcf7b5f62ce78f942adf891ecad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 23:50:57 +0000 Subject: [PATCH 04/27] chore: bump the patch group with 3 updates (#4706) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 88 +++++++++++++++++------------------ Cargo.toml | 2 +- prqlc/prqlc-macros/Cargo.toml | 2 +- prqlc/prqlc/Cargo.toml | 2 +- 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 844ba26f0dc2..7f6ad43f1e5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -334,7 +334,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -433,7 +433,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -489,7 +489,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", "syn_derive", ] @@ -770,7 +770,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -1121,7 +1121,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -1132,7 +1132,7 @@ checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" dependencies = [ "darling_core", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -1152,7 +1152,7 @@ checksum = "61bb5a1014ce6dfc2a378578509abe775a5aa06bff584a547555d9efdb81b926" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -1277,7 +1277,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -1297,7 +1297,7 @@ checksum = "5c785274071b1b420972453b306eeca06acf4633829db4223b58a2a8c5953bc4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -1441,7 +1441,7 @@ checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -1453,7 +1453,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -1465,7 +1465,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -1539,7 +1539,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -2229,9 +2229,9 @@ dependencies = [ [[package]] name = "minijinja" -version = "2.0.2" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e136ef580d7955019ab0a407b68d77c292a9976907e217900f3f76bc8f6dc1a4" +checksum = "933ee10775d58fca8238a84fe165dfe4bde8b07d7574f24d76ffea91170f3ac6" dependencies = [ "serde", ] @@ -2301,7 +2301,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", "termcolor", "thiserror", ] @@ -2554,7 +2554,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -2657,7 +2657,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -2924,7 +2924,7 @@ name = "prqlc-macros" version = "0.12.3" dependencies = [ "prqlc", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -3073,7 +3073,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -3086,7 +3086,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config 0.20.3", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -3169,7 +3169,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -3307,7 +3307,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.68", + "syn 2.0.69", "unicode-ident", ] @@ -3396,7 +3396,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -3500,7 +3500,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -3565,22 +3565,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -3591,7 +3591,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -3823,7 +3823,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -3836,7 +3836,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -3868,9 +3868,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.68" +version = "2.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "201fcda3845c23e8212cd466bfebf0bd20694490fc0356ae8e428e0824a915a6" dependencies = [ "proc-macro2", "quote", @@ -3886,7 +3886,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -3951,7 +3951,7 @@ checksum = "f9576037ae2a919664866c736156fd646d1c0e4e217244b1da05f56204dd6c61" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", "unicode-ident", ] @@ -3981,7 +3981,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -4115,7 +4115,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -4221,7 +4221,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -4455,7 +4455,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", "wasm-bindgen-shared", ] @@ -4489,7 +4489,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4522,7 +4522,7 @@ checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] @@ -4868,7 +4868,7 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.69", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b0b8d6a3f5fb..0883bde613fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,6 @@ itertools = "0.13.0" log = "0.4.22" schemars = "1.0.0-alpha.2" semver = {version = "1.0.23", features = ["serde"]} -serde = {version = "1.0.203", features = ["derive"]} +serde = {version = "1.0.204", features = ["derive"]} serde_json = "1.0.120" serde_yaml = {version = "0.9.34"} diff --git a/prqlc/prqlc-macros/Cargo.toml b/prqlc/prqlc-macros/Cargo.toml index adc9ed3e3b08..6fd7581ee355 100644 --- a/prqlc/prqlc-macros/Cargo.toml +++ b/prqlc/prqlc-macros/Cargo.toml @@ -16,7 +16,7 @@ test = false [dependencies] prqlc = {path = "../prqlc", default-features = false, version = "0.12.3"} -syn = "2.0.68" +syn = "2.0.69" [package.metadata.release] tag-name = "{{version}}" diff --git a/prqlc/prqlc/Cargo.toml b/prqlc/prqlc/Cargo.toml index 0599c992c02e..7a456257e6a6 100644 --- a/prqlc/prqlc/Cargo.toml +++ b/prqlc/prqlc/Cargo.toml @@ -84,7 +84,7 @@ clap_complete_fig = {version = "=4.4.2", optional = true} # We use minijinja just for the Jinja lexer, which is not part of the # public interface which is covered by semver guarantees. -minijinja = {version = "2.0.2", features = ["unstable_machinery"], optional = true} +minijinja = {version = "2.0.3", features = ["unstable_machinery"], optional = true} # For integration tests. These are gated by the `test-dbs` and `test-dbs-external` features, # rather than dev-dependencies, because dev-dependencies can't be optional. From 84d62c4fe38b83e88e9320455bd87db8f3760144 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 00:00:20 +0000 Subject: [PATCH 05/27] chore: bump oxsecurity/megalinter from 7.11.1 to 7.13.0 (#4704) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/lint-megalinter.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-megalinter.yaml b/.github/workflows/lint-megalinter.yaml index 97aa1fa6b455..315790151e6a 100644 --- a/.github/workflows/lint-megalinter.yaml +++ b/.github/workflows/lint-megalinter.yaml @@ -55,7 +55,7 @@ jobs: # You can override MegaLinter flavor used to have faster performances # More info at https://megalinter.io/flavors/ - uses: oxsecurity/megalinter@v7.11.1 + uses: oxsecurity/megalinter@v7.13.0 id: ml From 2b69b586dd54965e4c4d82fdb7decef4a1a51600 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Tue, 9 Jul 2024 09:02:21 -0700 Subject: [PATCH 06/27] internal: Add link to vendored code (#4709) --- prqlc/prqlc-parser/src/error/parse_error.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/prqlc/prqlc-parser/src/error/parse_error.rs b/prqlc/prqlc-parser/src/error/parse_error.rs index 3d0efcf696db..fac759f5c36c 100644 --- a/prqlc/prqlc-parser/src/error/parse_error.rs +++ b/prqlc/prqlc-parser/src/error/parse_error.rs @@ -270,6 +270,10 @@ impl From for Error { } } +// Vendored from +// https://github.com/zesterer/chumsky/pull/238/files#diff-97e25e2a0e41c578875856e97b659be2719a65227c104b992e3144efa000c35eR184 +// since it's private in chumsky + /// A type representing zero, one, or many labels applied to an error #[derive(Clone, Copy, Debug, PartialEq)] enum SimpleLabel { From 0b51fd3bd351487dd0a7a5251780034a322d16bf Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Tue, 9 Jul 2024 09:43:23 -0700 Subject: [PATCH 07/27] refactor: Small reorg of parser error (#4710) --- .../src/{error/mod.rs => error.rs} | 8 +- prqlc/prqlc-parser/src/error/test.rs | 114 ---------------- prqlc/prqlc-parser/src/parser/common.rs | 4 +- prqlc/prqlc-parser/src/parser/expr.rs | 2 +- .../prqlc-parser/src/parser/interpolation.rs | 2 +- prqlc/prqlc-parser/src/parser/mod.rs | 13 +- .../parse_error.rs => parser/perror.rs} | 124 +++++++++++++++++- prqlc/prqlc-parser/src/parser/stmt.rs | 2 +- prqlc/prqlc-parser/src/parser/types.rs | 2 +- 9 files changed, 136 insertions(+), 135 deletions(-) rename prqlc/prqlc-parser/src/{error/mod.rs => error.rs} (97%) delete mode 100644 prqlc/prqlc-parser/src/error/test.rs rename prqlc/prqlc-parser/src/{error/parse_error.rs => parser/perror.rs} (73%) diff --git a/prqlc/prqlc-parser/src/error/mod.rs b/prqlc/prqlc-parser/src/error.rs similarity index 97% rename from prqlc/prqlc-parser/src/error/mod.rs rename to prqlc/prqlc-parser/src/error.rs index b60997f00503..f9d255f856fd 100644 --- a/prqlc/prqlc-parser/src/error/mod.rs +++ b/prqlc/prqlc-parser/src/error.rs @@ -3,15 +3,9 @@ use std::fmt::Debug; use chumsky::error::Cheap; use serde::Serialize; -use crate::error::parse_error::PError; +use super::parser::perror::PError; use crate::span::Span; -/// Error message produced by the compiler. -pub mod parse_error; - -#[cfg(test)] -mod test; - /// A prqlc error. Used internally, exposed as prqlc::ErrorMessage. #[derive(Debug, Clone)] pub struct Error { diff --git a/prqlc/prqlc-parser/src/error/test.rs b/prqlc/prqlc-parser/src/error/test.rs deleted file mode 100644 index e0b39e529e7b..000000000000 --- a/prqlc/prqlc-parser/src/error/test.rs +++ /dev/null @@ -1,114 +0,0 @@ -use insta::{assert_debug_snapshot, assert_snapshot}; - -use crate::error::{Error, Errors, Reason, WithErrorInfo}; - -// Helper function to create a simple Error object -fn create_simple_error() -> Error { - Error::new_simple("A simple error message") - .push_hint("take a hint") - .with_code("E001") -} - -#[test] -fn display() { - assert_snapshot!(create_simple_error(), - @r###"Error { kind: Error, span: None, reason: Simple("A simple error message"), hints: ["take a hint"], code: Some("E001") }"### - ); - - let errors = Errors(vec![create_simple_error()]); - assert_snapshot!(errors, - @r###"Errors([Error { kind: Error, span: None, reason: Simple("A simple error message"), hints: ["take a hint"], code: Some("E001") }])"### - ); - assert_debug_snapshot!(errors, @r###" - Errors( - [ - Error { - kind: Error, - span: None, - reason: Simple( - "A simple error message", - ), - hints: [ - "take a hint", - ], - code: Some( - "E001", - ), - }, - ], - ) - "###) -} - -#[test] -fn test_simple_error() { - let err = create_simple_error(); - assert_debug_snapshot!(err, @r###" - Error { - kind: Error, - span: None, - reason: Simple( - "A simple error message", - ), - hints: [ - "take a hint", - ], - code: Some( - "E001", - ), - } - "###); -} - -#[test] -fn test_complex_error() { - assert_debug_snapshot!( - Error::new(Reason::Expected { - who: Some("Test".to_string()), - expected: "expected_value".to_string(), - found: "found_value".to_string(), - }) - .with_code("E002"), @r###" - Error { - kind: Error, - span: None, - reason: Expected { - who: Some( - "Test", - ), - expected: "expected_value", - found: "found_value", - }, - hints: [], - code: Some( - "E002", - ), - } - "###); -} - -#[test] -fn test_simple_error_with_result() { - let result: Result<(), Error> = Err(Error::new_simple("A simple error message")) - .with_hints(vec!["Take a hint"]) - .push_hint("Take another hint") - .with_code("E001"); - assert_debug_snapshot!(result, @r###" - Err( - Error { - kind: Error, - span: None, - reason: Simple( - "A simple error message", - ), - hints: [ - "Take a hint", - "Take another hint", - ], - code: Some( - "E001", - ), - }, - ) - "###); -} diff --git a/prqlc/prqlc-parser/src/parser/common.rs b/prqlc/prqlc-parser/src/parser/common.rs index 4536431afb54..95b522649302 100644 --- a/prqlc/prqlc-parser/src/parser/common.rs +++ b/prqlc/prqlc-parser/src/parser/common.rs @@ -1,8 +1,8 @@ use chumsky::prelude::*; -use crate::error::parse_error::PError; +use super::perror::PError; +use super::pr::{Annotation, Stmt, StmtKind}; use crate::lexer::lr::TokenKind; -use crate::parser::pr::{Annotation, Stmt, StmtKind}; use crate::span::Span; pub fn ident_part() -> impl Parser + Clone { diff --git a/prqlc/prqlc-parser/src/parser/expr.rs b/prqlc/prqlc-parser/src/parser/expr.rs index 0c19f537003a..6ab6b9e752d0 100644 --- a/prqlc/prqlc-parser/src/parser/expr.rs +++ b/prqlc/prqlc-parser/src/parser/expr.rs @@ -4,9 +4,9 @@ use chumsky::prelude::*; use itertools::Itertools; use super::interpolation; -use crate::error::parse_error::PError; use crate::lexer::lr::{Literal, TokenKind}; use crate::parser::common::{ctrl, ident_part, keyword, new_line}; +use crate::parser::perror::PError; use crate::parser::pr::Ident; use crate::parser::pr::*; use crate::parser::pr::{BinOp, UnOp}; diff --git a/prqlc/prqlc-parser/src/parser/interpolation.rs b/prqlc/prqlc-parser/src/parser/interpolation.rs index 0f957dbba46e..53c91dcc1828 100644 --- a/prqlc/prqlc-parser/src/parser/interpolation.rs +++ b/prqlc/prqlc-parser/src/parser/interpolation.rs @@ -1,8 +1,8 @@ use chumsky::prelude::*; use itertools::Itertools; -use crate::error::parse_error::{ChumError, PError}; use crate::lexer::lr::{Literal, TokenKind}; +use crate::parser::perror::{ChumError, PError}; use crate::parser::pr::*; use crate::span::{string_stream, Span}; diff --git a/prqlc/prqlc-parser/src/parser/mod.rs b/prqlc/prqlc-parser/src/parser/mod.rs index 89108237f04d..0bfcdfb9906e 100644 --- a/prqlc/prqlc-parser/src/parser/mod.rs +++ b/prqlc/prqlc-parser/src/parser/mod.rs @@ -1,18 +1,19 @@ +use chumsky::{prelude::*, Stream}; + +use crate::error::Error; +use crate::lexer::lr; +use crate::span::Span; + mod common; mod expr; mod interpolation; +pub(crate) mod perror; pub mod pr; pub(crate) mod stmt; #[cfg(test)] mod test; mod types; -use chumsky::{prelude::*, Stream}; - -use crate::error::Error; -use crate::lexer::lr; -use crate::span::Span; - pub fn parse_lr_to_pr( source: &str, source_id: u16, diff --git a/prqlc/prqlc-parser/src/error/parse_error.rs b/prqlc/prqlc-parser/src/parser/perror.rs similarity index 73% rename from prqlc/prqlc-parser/src/error/parse_error.rs rename to prqlc/prqlc-parser/src/parser/perror.rs index fac759f5c36c..35056a5f091b 100644 --- a/prqlc/prqlc-parser/src/error/parse_error.rs +++ b/prqlc/prqlc-parser/src/parser/perror.rs @@ -1,9 +1,11 @@ use core::fmt; use std::collections::HashSet; -use std::fmt::{Debug, Display}; +use std::fmt::Debug; +use std::fmt::Display; use std::hash::Hash; -use crate::error::{Error, ErrorSource, Reason, WithErrorInfo}; +use crate::error::WithErrorInfo; +use crate::error::{Error, ErrorSource, Reason}; use crate::lexer::lr::TokenKind; use crate::span::Span; @@ -303,3 +305,121 @@ impl From for Option<&'static str> { } } } + +#[cfg(test)] +mod tests { + use insta::{assert_debug_snapshot, assert_snapshot}; + + use crate::error::{Error, Errors, Reason, WithErrorInfo}; + + // Helper function to create a simple Error object + fn create_simple_error() -> Error { + Error::new_simple("A simple error message") + .push_hint("take a hint") + .with_code("E001") + } + + #[test] + fn display() { + assert_snapshot!(create_simple_error(), + @r###"Error { kind: Error, span: None, reason: Simple("A simple error message"), hints: ["take a hint"], code: Some("E001") }"### + ); + + let errors = Errors(vec![create_simple_error()]); + assert_snapshot!(errors, + @r###"Errors([Error { kind: Error, span: None, reason: Simple("A simple error message"), hints: ["take a hint"], code: Some("E001") }])"### + ); + assert_debug_snapshot!(errors, @r###" + Errors( + [ + Error { + kind: Error, + span: None, + reason: Simple( + "A simple error message", + ), + hints: [ + "take a hint", + ], + code: Some( + "E001", + ), + }, + ], + ) + "###) + } + + #[test] + fn test_simple_error() { + let err = create_simple_error(); + assert_debug_snapshot!(err, @r###" + Error { + kind: Error, + span: None, + reason: Simple( + "A simple error message", + ), + hints: [ + "take a hint", + ], + code: Some( + "E001", + ), + } + "###); + } + + #[test] + fn test_complex_error() { + assert_debug_snapshot!( + Error::new(Reason::Expected { + who: Some("Test".to_string()), + expected: "expected_value".to_string(), + found: "found_value".to_string(), + }) + .with_code("E002"), @r###" + Error { + kind: Error, + span: None, + reason: Expected { + who: Some( + "Test", + ), + expected: "expected_value", + found: "found_value", + }, + hints: [], + code: Some( + "E002", + ), + } + "###); + } + + #[test] + fn test_simple_error_with_result() { + let result: Result<(), Error> = Err(Error::new_simple("A simple error message")) + .with_hints(vec!["Take a hint"]) + .push_hint("Take another hint") + .with_code("E001"); + assert_debug_snapshot!(result, @r###" + Err( + Error { + kind: Error, + span: None, + reason: Simple( + "A simple error message", + ), + hints: [ + "Take a hint", + "Take another hint", + ], + code: Some( + "E001", + ), + }, + ) + "###); + } +} diff --git a/prqlc/prqlc-parser/src/parser/stmt.rs b/prqlc/prqlc-parser/src/parser/stmt.rs index 17394ebefe27..37b9fcfda8d2 100644 --- a/prqlc/prqlc-parser/src/parser/stmt.rs +++ b/prqlc/prqlc-parser/src/parser/stmt.rs @@ -6,8 +6,8 @@ use semver::VersionReq; use super::common::{ctrl, ident_part, into_stmt, keyword, new_line}; use super::expr::{expr, expr_call, ident, pipeline}; -use crate::error::parse_error::PError; use crate::lexer::lr::{Literal, TokenKind}; +use crate::parser::perror::PError; use crate::parser::pr::*; use crate::parser::types::type_expr; diff --git a/prqlc/prqlc-parser/src/parser/types.rs b/prqlc/prqlc-parser/src/parser/types.rs index c4d74fc69f3b..c4381e6d9dc3 100644 --- a/prqlc/prqlc-parser/src/parser/types.rs +++ b/prqlc/prqlc-parser/src/parser/types.rs @@ -1,9 +1,9 @@ use chumsky::prelude::*; use super::common::*; -use crate::error::parse_error::PError; use crate::lexer::lr::TokenKind; use crate::parser::expr::ident; +use crate::parser::perror::PError; use crate::parser::pr::*; pub fn type_expr() -> impl Parser + Clone { From ca95ac2c31474d249d5e24a41996b6adb7117a9b Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Tue, 9 Jul 2024 09:59:16 -0700 Subject: [PATCH 08/27] refactor: Small collapse of single file `mod.rs` in lexer (#4711) --- prqlc/prqlc-parser/src/lexer/{lr/token.rs => lr.rs} | 6 +++++- prqlc/prqlc-parser/src/lexer/lr/mod.rs | 8 -------- prqlc/prqlc-parser/src/lexer/mod.rs | 2 +- prqlc/prqlc-parser/src/parser/types.rs | 6 +++--- 4 files changed, 9 insertions(+), 13 deletions(-) rename prqlc/prqlc-parser/src/lexer/{lr/token.rs => lr.rs} (98%) delete mode 100644 prqlc/prqlc-parser/src/lexer/lr/mod.rs diff --git a/prqlc/prqlc-parser/src/lexer/lr/token.rs b/prqlc/prqlc-parser/src/lexer/lr.rs similarity index 98% rename from prqlc/prqlc-parser/src/lexer/lr/token.rs rename to prqlc/prqlc-parser/src/lexer/lr.rs index d900917e42d1..1fa9e968d1a0 100644 --- a/prqlc/prqlc-parser/src/lexer/lr/token.rs +++ b/prqlc/prqlc-parser/src/lexer/lr.rs @@ -1,6 +1,10 @@ +use serde::{Deserialize, Serialize}; + use enum_as_inner::EnumAsInner; use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct Tokens(pub Vec); #[derive(Clone, PartialEq, Serialize, Deserialize, Eq, JsonSchema)] pub struct Token { diff --git a/prqlc/prqlc-parser/src/lexer/lr/mod.rs b/prqlc/prqlc-parser/src/lexer/lr/mod.rs deleted file mode 100644 index cec8ac837c5e..000000000000 --- a/prqlc/prqlc-parser/src/lexer/lr/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -mod token; - -pub use token::*; - -use serde::{Deserialize, Serialize}; - -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct Tokens(pub Vec); diff --git a/prqlc/prqlc-parser/src/lexer/mod.rs b/prqlc/prqlc-parser/src/lexer/mod.rs index 4ecbdaa7549d..cc3f07e474f2 100644 --- a/prqlc/prqlc-parser/src/lexer/mod.rs +++ b/prqlc/prqlc-parser/src/lexer/mod.rs @@ -2,8 +2,8 @@ use chumsky::error::Cheap; use chumsky::prelude::*; use chumsky::text::{newline, Character}; +use self::lr::{Literal, Token, TokenKind, ValueAndUnit}; use crate::error::{Error, ErrorSource, Reason, WithErrorInfo}; -use crate::lexer::lr::{Literal, Token, TokenKind, ValueAndUnit}; use crate::span::Span; pub mod lr; diff --git a/prqlc/prqlc-parser/src/parser/types.rs b/prqlc/prqlc-parser/src/parser/types.rs index c4381e6d9dc3..b960eb0ca10a 100644 --- a/prqlc/prqlc-parser/src/parser/types.rs +++ b/prqlc/prqlc-parser/src/parser/types.rs @@ -1,10 +1,10 @@ use chumsky::prelude::*; use super::common::*; +use super::expr::ident; +use super::perror::PError; +use super::pr::*; use crate::lexer::lr::TokenKind; -use crate::parser::expr::ident; -use crate::parser::perror::PError; -use crate::parser::pr::*; pub fn type_expr() -> impl Parser + Clone { recursive(|nested_type_expr| { From 03ea8fe6e57981ebf64881d471264f6c5b006086 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Tue, 9 Jul 2024 21:27:21 -0700 Subject: [PATCH 09/27] refactor: Move newline consumption out of pipeline (#4712) --- prqlc/prqlc-parser/src/parser/expr.rs | 9 ++++-- prqlc/prqlc-parser/src/parser/stmt.rs | 1 + prqlc/prqlc-parser/src/test.rs | 41 +++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/prqlc/prqlc-parser/src/parser/expr.rs b/prqlc/prqlc-parser/src/parser/expr.rs index 6ab6b9e752d0..58d8bafb7b6b 100644 --- a/prqlc/prqlc-parser/src/parser/expr.rs +++ b/prqlc/prqlc-parser/src/parser/expr.rs @@ -123,7 +123,8 @@ fn array( fn pipeline_expr( expr: impl Parser + Clone, ) -> impl Parser + Clone { - expr.delimited_by(ctrl('('), ctrl(')')) + expr.then_ignore(new_line().repeated()) + .delimited_by(ctrl('('), ctrl(')')) .recover_with(nested_delimiters( TokenKind::Control('('), TokenKind::Control(')'), @@ -268,7 +269,10 @@ where // expr has to be a param, because it can be either a normal expr() or a // recursive expr called from within expr(), which causes a stack overflow - let pipe = ctrl('|').or(new_line().repeated().at_least(1).ignored()); + let pipe = choice(( + ctrl('|').ignored(), + new_line().repeated().at_least(1).ignored(), + )); new_line() .repeated() @@ -292,7 +296,6 @@ where }) }), ) - .then_ignore(new_line().repeated()) .labelled("pipeline") } diff --git a/prqlc/prqlc-parser/src/parser/stmt.rs b/prqlc/prqlc-parser/src/parser/stmt.rs index 37b9fcfda8d2..e1fe38301e3d 100644 --- a/prqlc/prqlc-parser/src/parser/stmt.rs +++ b/prqlc/prqlc-parser/src/parser/stmt.rs @@ -128,6 +128,7 @@ fn var_def() -> impl Parser + Clone { .labelled("variable definition"); let main_or_into = pipeline(expr_call()) + .then_ignore(new_line().repeated()) .map(Box::new) .then(keyword("into").ignore_then(ident_part()).or_not()) .map(|(value, name)| { diff --git a/prqlc/prqlc-parser/src/test.rs b/prqlc/prqlc-parser/src/test.rs index c49ffdf4826e..70ab97f1a175 100644 --- a/prqlc/prqlc-parser/src/test.rs +++ b/prqlc/prqlc-parser/src/test.rs @@ -2349,6 +2349,47 @@ fn test_multiline_string() { "### ) } +#[test] +fn test_empty_lines() { + // The span of the Pipeline shouldn't include the empty lines; the VarDef + // should have a larger span + assert_yaml_snapshot!(parse_single(r#" +from artists +derive x = 5 + + + +"#).unwrap(), @r###" + --- + - VarDef: + kind: Main + name: main + value: + Pipeline: + exprs: + - FuncCall: + name: + Ident: from + span: "0:1-5" + args: + - Ident: artists + span: "0:6-13" + span: "0:1-13" + - FuncCall: + name: + Ident: derive + span: "0:14-20" + args: + - Literal: + Integer: 5 + span: "0:25-26" + alias: x + span: "0:14-26" + span: "0:1-26" + span: "0:1-31" + "### ) +} + #[test] fn test_coalesce() { assert_yaml_snapshot!(parse_single(r###" From 8eff9c955b6b72c5c8de20094607b8bf7937e5a0 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Wed, 10 Jul 2024 00:06:43 -0700 Subject: [PATCH 10/27] tweak(refactor): Move filtering tokens into `prepare_stream` (#4713) --- prqlc/prqlc-parser/src/parser/mod.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/prqlc/prqlc-parser/src/parser/mod.rs b/prqlc/prqlc-parser/src/parser/mod.rs index 0bfcdfb9906e..e16221e28793 100644 --- a/prqlc/prqlc-parser/src/parser/mod.rs +++ b/prqlc/prqlc-parser/src/parser/mod.rs @@ -17,18 +17,11 @@ mod types; pub fn parse_lr_to_pr( source: &str, source_id: u16, - lr_iter: Vec, + lr: Vec, ) -> (Option>, Vec) { // We don't want comments in the AST (but we do intend to use them as part of // formatting) - let semantic_tokens = lr_iter.into_iter().filter(|token| { - !matches!( - token.kind, - lr::TokenKind::Comment(_) | lr::TokenKind::LineWrap(_) | lr::TokenKind::DocComment(_) - ) - }); - - let stream = prepare_stream(semantic_tokens, source, source_id); + let stream = prepare_stream(lr.into_iter(), source, source_id); let (pr, parse_errors) = ::chumsky::Parser::parse_recovery(&stmt::source(), stream); let errors = parse_errors.into_iter().map(|e| e.into()).collect(); @@ -39,12 +32,21 @@ pub fn parse_lr_to_pr( /// Convert the output of the lexer into the input of the parser. Requires /// supplying the original source code. -fn prepare_stream( +pub(crate) fn prepare_stream( tokens: impl Iterator, source: &str, source_id: u16, ) -> Stream + Sized> { - let tokens = tokens + // We don't want comments in the AST (but we do intend to use them as part of + // formatting) + let semantic_tokens = tokens.filter(|token| { + !matches!( + token.kind, + lr::TokenKind::Comment(_) | lr::TokenKind::LineWrap(_) | lr::TokenKind::DocComment(_) + ) + }); + + let tokens = semantic_tokens .into_iter() .map(move |token| (token.kind, Span::new(source_id, token.span))); let len = source.chars().count(); From a1ba7ce06647e9c6efa7198659253a27a3508660 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Wed, 10 Jul 2024 00:15:13 -0700 Subject: [PATCH 11/27] tweak: Adjust name of repr in docstring (#4714) --- prqlc/prqlc-parser/src/parser/mod.rs | 2 +- prqlc/prqlc/src/parser.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/prqlc/prqlc-parser/src/parser/mod.rs b/prqlc/prqlc-parser/src/parser/mod.rs index e16221e28793..0e4c2798a3b3 100644 --- a/prqlc/prqlc-parser/src/parser/mod.rs +++ b/prqlc/prqlc-parser/src/parser/mod.rs @@ -22,7 +22,7 @@ pub fn parse_lr_to_pr( // We don't want comments in the AST (but we do intend to use them as part of // formatting) let stream = prepare_stream(lr.into_iter(), source, source_id); - let (pr, parse_errors) = ::chumsky::Parser::parse_recovery(&stmt::source(), stream); + let (pr, parse_errors) = stmt::source().parse_recovery(stream); let errors = parse_errors.into_iter().map(|e| e.into()).collect(); log::debug!("parse errors: {errors:?}"); diff --git a/prqlc/prqlc/src/parser.rs b/prqlc/prqlc/src/parser.rs index 95d9cc472d5d..55774272d917 100644 --- a/prqlc/prqlc/src/parser.rs +++ b/prqlc/prqlc/src/parser.rs @@ -52,7 +52,7 @@ pub fn parse(file_tree: &SourceTree) -> Result { } } -/// Build PRQL AST from a PRQL query string. +/// Build PR AST from a PRQL query string. pub(crate) fn parse_source(source: &str, source_id: u16) -> Result, Vec> { let (tokens, mut errors) = prqlc_parser::lexer::lex_source_recovery(source, source_id); From 4fc6b55ba47bf7c800182a4b34fdd6987f57ac92 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Wed, 10 Jul 2024 10:11:44 -0700 Subject: [PATCH 12/27] refactor: Use a `parse_with_parser` func to test any parser (#4715) --- prqlc/prqlc-parser/src/lib.rs | 2 +- prqlc/prqlc-parser/src/parser/test.rs | 1310 ++++++++++++++++++++++- prqlc/prqlc-parser/src/test.rs | 1372 +------------------------ 3 files changed, 1356 insertions(+), 1328 deletions(-) diff --git a/prqlc/prqlc-parser/src/lib.rs b/prqlc/prqlc-parser/src/lib.rs index 3d66e52cf158..18ddc7755db8 100644 --- a/prqlc/prqlc-parser/src/lib.rs +++ b/prqlc/prqlc-parser/src/lib.rs @@ -4,4 +4,4 @@ pub mod lexer; pub mod parser; pub mod span; #[cfg(test)] -mod test; +pub(crate) mod test; diff --git a/prqlc/prqlc-parser/src/parser/test.rs b/prqlc/prqlc-parser/src/parser/test.rs index 842a0610edde..0d6ff9963f0f 100644 --- a/prqlc/prqlc-parser/src/parser/test.rs +++ b/prqlc/prqlc-parser/src/parser/test.rs @@ -1,7 +1,16 @@ -use crate::lexer::lex_source; -use crate::lexer::lr::TokenKind; +use insta::assert_yaml_snapshot; + use crate::parser::prepare_stream; use crate::span::Span; +use crate::test::parse_with_parser; +use crate::{error::Error, lexer::lex_source}; +use crate::{lexer::lr::TokenKind, parser::pr::FuncCall}; + +use super::pr::Expr; + +fn parse_expr(source: &str) -> Result> { + parse_with_parser(source, super::expr::expr_call()) +} #[test] fn test_prepare_stream() { @@ -30,3 +39,1300 @@ fn test_prepare_stream() { - "0:30-36" "###); } + +#[test] +fn test_ranges() { + assert_yaml_snapshot!(parse_expr(r#"3..5"#).unwrap(), @r###" + --- + Range: + start: + Literal: + Integer: 3 + span: "0:0-1" + end: + Literal: + Integer: 5 + span: "0:3-4" + span: "0:0-4" + "###); + + assert_yaml_snapshot!(parse_expr(r#"-2..-5"#).unwrap(), @r###" + --- + Range: + start: + Unary: + op: Neg + expr: + Literal: + Integer: 2 + span: "0:1-2" + span: "0:0-2" + end: + Unary: + op: Neg + expr: + Literal: + Integer: 5 + span: "0:5-6" + span: "0:4-6" + span: "0:0-6" + "###); + + assert_yaml_snapshot!(parse_expr(r#"(-2..(-5 | abs))"#).unwrap(), @r###" + --- + Range: + start: + Unary: + op: Neg + expr: + Literal: + Integer: 2 + span: "0:2-3" + span: "0:1-3" + end: + Pipeline: + exprs: + - Unary: + op: Neg + expr: + Literal: + Integer: 5 + span: "0:7-8" + span: "0:6-8" + - Ident: abs + span: "0:11-14" + span: "0:6-14" + span: "0:0-16" + "###); + + assert_yaml_snapshot!(parse_expr(r#"(2 + 5)..'a'"#).unwrap(), @r###" + --- + Range: + start: + Binary: + left: + Literal: + Integer: 2 + span: "0:1-2" + op: Add + right: + Literal: + Integer: 5 + span: "0:5-6" + span: "0:1-6" + end: + Literal: + String: a + span: "0:9-12" + span: "0:0-12" + "###); + + assert_yaml_snapshot!(parse_expr(r#"1.6..rel.col"#).unwrap(), @r###" + --- + Range: + start: + Literal: + Float: 1.6 + span: "0:0-3" + end: + Indirection: + base: + Ident: rel + span: "0:5-8" + field: + Name: col + span: "0:8-12" + span: "0:0-12" + "###); + + assert_yaml_snapshot!(parse_expr(r#"6.."#).unwrap(), @r###" + --- + Range: + start: + Literal: + Integer: 6 + span: "0:0-1" + end: ~ + span: "0:0-3" + "###); + assert_yaml_snapshot!(parse_expr(r#"..7"#).unwrap(), @r###" + --- + Range: + start: ~ + end: + Literal: + Integer: 7 + span: "0:2-3" + span: "0:0-3" + "###); + + assert_yaml_snapshot!(parse_expr(r#".."#).unwrap(), @r###" + --- + Range: + start: ~ + end: ~ + span: "0:0-2" + "###); + + assert_yaml_snapshot!(parse_expr(r#"@2020-01-01..@2021-01-01"#).unwrap(), @r###" + --- + Range: + start: + Literal: + Date: 2020-01-01 + span: "0:0-11" + end: + Literal: + Date: 2021-01-01 + span: "0:13-24" + span: "0:0-24" + "###); +} + +#[test] +fn test_basic_exprs() { + assert_yaml_snapshot!(parse_expr(r#"country == "USA""#).unwrap(), @r###" + --- + Binary: + left: + Ident: country + span: "0:0-7" + op: Eq + right: + Literal: + String: USA + span: "0:11-16" + span: "0:0-16" + "###); + assert_yaml_snapshot!(parse_expr("select {a, b, c}").unwrap(), @r###" + --- + FuncCall: + name: + Ident: select + span: "0:0-6" + args: + - Tuple: + - Ident: a + span: "0:8-9" + - Ident: b + span: "0:11-12" + - Ident: c + span: "0:14-15" + span: "0:7-16" + span: "0:0-16" + "###); + assert_yaml_snapshot!(parse_expr( + "group {title, country} ( + aggregate {sum salary} + )" + ).unwrap(), @r###" + --- + FuncCall: + name: + Ident: group + span: "0:0-5" + args: + - Tuple: + - Ident: title + span: "0:7-12" + - Ident: country + span: "0:14-21" + span: "0:6-22" + - FuncCall: + name: + Ident: aggregate + span: "0:41-50" + args: + - Tuple: + - FuncCall: + name: + Ident: sum + span: "0:52-55" + args: + - Ident: salary + span: "0:56-62" + span: "0:52-62" + span: "0:51-63" + span: "0:41-63" + span: "0:0-77" + "###); + assert_yaml_snapshot!(parse_expr( + r#" filter country == "USA""# + ).unwrap(), @r###" + --- + FuncCall: + name: + Ident: filter + span: "0:4-10" + args: + - Binary: + left: + Ident: country + span: "0:11-18" + op: Eq + right: + Literal: + String: USA + span: "0:22-27" + span: "0:11-27" + span: "0:4-27" + "###); + assert_yaml_snapshot!(parse_expr("{a, b, c,}").unwrap(), @r###" + --- + Tuple: + - Ident: a + span: "0:1-2" + - Ident: b + span: "0:4-5" + - Ident: c + span: "0:7-8" + span: "0:0-10" + "###); + assert_yaml_snapshot!(parse_expr( + r#"{ + gross_salary = salary + payroll_tax, + gross_cost = gross_salary + benefits_cost +}"# + ).unwrap(), @r###" + --- + Tuple: + - Binary: + left: + Ident: salary + span: "0:19-25" + op: Add + right: + Ident: payroll_tax + span: "0:28-39" + span: "0:19-39" + alias: gross_salary + - Binary: + left: + Ident: gross_salary + span: "0:58-70" + op: Add + right: + Ident: benefits_cost + span: "0:73-86" + span: "0:58-86" + alias: gross_cost + span: "0:0-88" + "###); + + assert_yaml_snapshot!(parse_expr( + "join side:left country (id==employee_id)" + ).unwrap(), @r###" + --- + FuncCall: + name: + Ident: join + span: "0:0-4" + args: + - Ident: country + span: "0:15-22" + - Binary: + left: + Ident: id + span: "0:24-26" + op: Eq + right: + Ident: employee_id + span: "0:28-39" + span: "0:24-39" + named_args: + side: + Ident: left + span: "0:10-14" + span: "0:0-40" + "###); + assert_yaml_snapshot!(parse_expr("1 + 2").unwrap(), @r###" + --- + Binary: + left: + Literal: + Integer: 1 + span: "0:0-1" + op: Add + right: + Literal: + Integer: 2 + span: "0:5-6" + span: "0:0-6" + "###) +} + +#[test] +fn test_string() { + let double_quoted_ast = parse_expr(r#"" U S A ""#).unwrap(); + assert_yaml_snapshot!(double_quoted_ast, @r###" + --- + Literal: + String: " U S A " + span: "0:0-9" + "###); + + let single_quoted_ast = parse_expr(r#"' U S A '"#).unwrap(); + assert_eq!(single_quoted_ast, double_quoted_ast); + + // Single quotes within double quotes should produce a string containing + // the single quotes (and vice versa). + assert_yaml_snapshot!(parse_expr(r#""' U S A '""#).unwrap(), @r###" + --- + Literal: + String: "' U S A '" + span: "0:0-11" + "###); + assert_yaml_snapshot!(parse_expr(r#"'" U S A "'"#).unwrap(), @r###" + --- + Literal: + String: "\" U S A \"" + span: "0:0-11" + "###); + + parse_expr(r#"" U S A"#).unwrap_err(); + parse_expr(r#"" U S A '"#).unwrap_err(); + + assert_yaml_snapshot!(parse_expr(r#"" \nU S A ""#).unwrap(), @r###" + --- + Literal: + String: " \nU S A " + span: "0:0-11" + "###); + + assert_yaml_snapshot!(parse_expr(r#"r" \nU S A ""#).unwrap(), @r###" + --- + Literal: + String: " \\nU S A " + span: "0:0-12" + "###); + + let multi_double = parse_expr( + r#"""" +'' +Canada +" + +""""#, + ) + .unwrap(); + assert_yaml_snapshot!(multi_double, @r###" + --- + Literal: + String: "\n''\nCanada\n\"\n\n" + span: "0:0-20" + "###); + + let multi_single = parse_expr( + r#"''' +Canada +" +""" + +'''"#, + ) + .unwrap(); + assert_yaml_snapshot!(multi_single, @r###" + --- + Literal: + String: "\nCanada\n\"\n\"\"\"\n\n" + span: "0:0-21" + "###); + + assert_yaml_snapshot!( + parse_expr("''").unwrap(), + @r###" + --- + Literal: + String: "" + span: "0:0-2" + "###); +} + +#[test] +fn test_s_string() { + assert_yaml_snapshot!(parse_expr(r#"s"SUM({col})""#).unwrap(), @r###" + --- + SString: + - String: SUM( + - Expr: + expr: + Ident: col + span: "0:7-10" + format: ~ + - String: ) + span: "0:0-13" + "###); + assert_yaml_snapshot!(parse_expr(r#"s"SUM({rel.`Col name`})""#).unwrap(), @r###" + --- + SString: + - String: SUM( + - Expr: + expr: + Indirection: + base: + Ident: rel + span: "0:7-10" + field: + Name: Col name + span: "0:11-21" + format: ~ + - String: ) + span: "0:0-24" + "###) +} + +#[test] +fn test_s_string_braces() { + assert_yaml_snapshot!(parse_expr(r#"s"{{?crystal_var}}""#).unwrap(), @r###" + --- + SString: + - String: "{?crystal_var}" + span: "0:0-19" + "###); + assert_yaml_snapshot!(parse_expr(r#"s"foo{{bar""#).unwrap(), @r###" + --- + SString: + - String: "foo{bar" + span: "0:0-11" + "###); + parse_expr(r#"s"foo{{bar}""#).unwrap_err(); +} + +#[test] +fn test_tuple() { + assert_yaml_snapshot!(parse_expr(r#"{1 + 1, 2}"#).unwrap(), @r###" + --- + Tuple: + - Binary: + left: + Literal: + Integer: 1 + span: "0:1-2" + op: Add + right: + Literal: + Integer: 1 + span: "0:5-6" + span: "0:1-6" + - Literal: + Integer: 2 + span: "0:8-9" + span: "0:0-10" + "###); + assert_yaml_snapshot!(parse_expr(r#"{1 + (f 1), 2}"#).unwrap(), @r###" + --- + Tuple: + - Binary: + left: + Literal: + Integer: 1 + span: "0:1-2" + op: Add + right: + FuncCall: + name: + Ident: f + span: "0:6-7" + args: + - Literal: + Integer: 1 + span: "0:8-9" + span: "0:6-9" + span: "0:1-10" + - Literal: + Integer: 2 + span: "0:12-13" + span: "0:0-14" + "###); + // Line breaks + assert_yaml_snapshot!(parse_expr( + r#"{1, + + 2}"# + ).unwrap(), @r###" + --- + Tuple: + - Literal: + Integer: 1 + span: "0:1-2" + - Literal: + Integer: 2 + span: "0:21-22" + span: "0:0-23" + "###); + // Function call in a tuple + let ab = parse_expr(r#"{a b}"#).unwrap(); + let a_comma_b = parse_expr(r#"{a, b}"#).unwrap(); + assert_yaml_snapshot!(ab, @r###" + --- + Tuple: + - FuncCall: + name: + Ident: a + span: "0:1-2" + args: + - Ident: b + span: "0:3-4" + span: "0:1-4" + span: "0:0-5" + "###); + assert_yaml_snapshot!(a_comma_b, @r###" + --- + Tuple: + - Ident: a + span: "0:1-2" + - Ident: b + span: "0:4-5" + span: "0:0-6" + "###); + assert_ne!(ab, a_comma_b); + + assert_yaml_snapshot!(parse_expr(r#"{amount, +amount, -amount}"#).unwrap(), @r###" + --- + Tuple: + - Ident: amount + span: "0:1-7" + - Unary: + op: Add + expr: + Ident: amount + span: "0:10-16" + span: "0:9-16" + - Unary: + op: Neg + expr: + Ident: amount + span: "0:19-25" + span: "0:18-25" + span: "0:0-26" + "###); + // Operators in tuple items + assert_yaml_snapshot!(parse_expr(r#"{amount, +amount, -amount}"#).unwrap(), @r###" + --- + Tuple: + - Ident: amount + span: "0:1-7" + - Unary: + op: Add + expr: + Ident: amount + span: "0:10-16" + span: "0:9-16" + - Unary: + op: Neg + expr: + Ident: amount + span: "0:19-25" + span: "0:18-25" + span: "0:0-26" + "###); +} + +#[test] +fn test_number() { + assert_yaml_snapshot!(parse_expr(r#"23"#).unwrap(), @r###" + --- + Literal: + Integer: 23 + span: "0:0-2" + "###); + assert_yaml_snapshot!(parse_expr(r#"2_3_4.5_6"#).unwrap(), @r###" + --- + Literal: + Float: 234.56 + span: "0:0-9" + "###); + assert_yaml_snapshot!(parse_expr(r#"23.6"#).unwrap(), @r###" + --- + Literal: + Float: 23.6 + span: "0:0-4" + "###); + assert_yaml_snapshot!(parse_expr(r#"23.0"#).unwrap(), @r###" + --- + Literal: + Float: 23 + span: "0:0-4" + "###); + assert_yaml_snapshot!(parse_expr(r#"2 + 2"#).unwrap(), @r###" + --- + Binary: + left: + Literal: + Integer: 2 + span: "0:0-1" + op: Add + right: + Literal: + Integer: 2 + span: "0:4-5" + span: "0:0-5" + "###); + + // Underscores at the beginning are parsed as ident + assert!(parse_expr("_2").unwrap().kind.into_ident().is_ok()); + assert!(parse_expr("_").unwrap().kind.into_ident().is_ok()); + + assert!(parse_expr("_2.3").unwrap().kind.is_indirection()); + + assert_yaml_snapshot!(parse_expr(r#"2e3"#).unwrap(), @r###" + --- + Literal: + Float: 2000 + span: "0:0-3" + "###); + + // expr_of_string("2_").unwrap_err(); // TODO + // expr_of_string("2.3_").unwrap_err(); // TODO +} + +#[test] +fn test_derive() { + assert_yaml_snapshot!( + parse_expr(r#"derive {x = 5, y = (-x)}"#).unwrap() + , @r###" + --- + FuncCall: + name: + Ident: derive + span: "0:0-6" + args: + - Tuple: + - Literal: + Integer: 5 + span: "0:12-13" + alias: x + - Unary: + op: Neg + expr: + Ident: x + span: "0:21-22" + span: "0:19-23" + alias: y + span: "0:7-24" + span: "0:0-24" + "###); +} + +#[test] +fn test_select() { + assert_yaml_snapshot!( + parse_expr(r#"select x"#).unwrap() + , @r###" + --- + FuncCall: + name: + Ident: select + span: "0:0-6" + args: + - Ident: x + span: "0:7-8" + span: "0:0-8" + "###); + + assert_yaml_snapshot!( + parse_expr(r#"select !{x}"#).unwrap() + , @r###" + --- + FuncCall: + name: + Ident: select + span: "0:0-6" + args: + - Unary: + op: Not + expr: + Tuple: + - Ident: x + span: "0:9-10" + span: "0:8-11" + span: "0:7-11" + span: "0:0-11" + "###); + + assert_yaml_snapshot!( + parse_expr(r#"select {x, y}"#).unwrap() + , @r###" + --- + FuncCall: + name: + Ident: select + span: "0:0-6" + args: + - Tuple: + - Ident: x + span: "0:8-9" + - Ident: y + span: "0:11-12" + span: "0:7-13" + span: "0:0-13" + "###); +} + +#[test] +fn test_expr() { + assert_yaml_snapshot!( + parse_expr(r#"country == "USA""#).unwrap() + , @r###" + --- + Binary: + left: + Ident: country + span: "0:0-7" + op: Eq + right: + Literal: + String: USA + span: "0:11-16" + span: "0:0-16" + "###); + assert_yaml_snapshot!(parse_expr( + r#"{ + gross_salary = salary + payroll_tax, + gross_cost = gross_salary + benefits_cost, +}"#).unwrap(), @r###" + --- + Tuple: + - Binary: + left: + Ident: salary + span: "0:19-25" + op: Add + right: + Ident: payroll_tax + span: "0:28-39" + span: "0:19-39" + alias: gross_salary + - Binary: + left: + Ident: gross_salary + span: "0:58-70" + op: Add + right: + Ident: benefits_cost + span: "0:73-86" + span: "0:58-86" + alias: gross_cost + span: "0:0-89" + "###); + assert_yaml_snapshot!( + parse_expr( + "(salary + payroll_tax) * (1 + tax_rate)" + ).unwrap(), + @r###" + --- + Binary: + left: + Binary: + left: + Ident: salary + span: "0:1-7" + op: Add + right: + Ident: payroll_tax + span: "0:10-21" + span: "0:1-21" + op: Mul + right: + Binary: + left: + Literal: + Integer: 1 + span: "0:26-27" + op: Add + right: + Ident: tax_rate + span: "0:30-38" + span: "0:26-38" + span: "0:0-39" + "###); +} + +#[test] +fn test_regex() { + assert_yaml_snapshot!( + parse_expr( + "'oba' ~= 'foobar'" + ).unwrap(), + @r###" + --- + Binary: + left: + Literal: + String: oba + span: "0:0-5" + op: RegexSearch + right: + Literal: + String: foobar + span: "0:9-17" + span: "0:0-17" + "###); +} + +#[test] +fn test_func_call() { + // Function without argument + let ast = parse_expr(r#"count"#).unwrap(); + let ident = ast.kind.into_ident().unwrap(); + assert_yaml_snapshot!( + ident, @r###" + --- + count + "###); + + let ast = parse_expr(r#"s 'foo'"#).unwrap(); + assert_yaml_snapshot!( + ast, @r###" + --- + FuncCall: + name: + Ident: s + span: "0:0-1" + args: + - Literal: + String: foo + span: "0:2-7" + span: "0:0-7" + "###); + + // A non-friendly option for #154 + let ast = parse_expr(r#"count s'*'"#).unwrap(); + let func_call: FuncCall = ast.kind.into_func_call().unwrap(); + assert_yaml_snapshot!( + func_call, @r###" + --- + name: + Ident: count + span: "0:0-5" + args: + - SString: + - String: "*" + span: "0:6-10" + "###); + + parse_expr("plus_one x:0 x:0 ").unwrap_err(); + + let ast = parse_expr(r#"add bar to=3"#).unwrap(); + assert_yaml_snapshot!( + ast, @r###" + --- + FuncCall: + name: + Ident: add + span: "0:0-3" + args: + - Ident: bar + span: "0:4-7" + - Literal: + Integer: 3 + span: "0:11-12" + alias: to + span: "0:0-12" + "###); +} + +#[test] +fn test_right_assoc() { + assert_yaml_snapshot!(parse_expr(r#"2 ** 3 ** 4"#).unwrap(), @r###" + --- + Binary: + left: + Literal: + Integer: 2 + span: "0:0-1" + op: Pow + right: + Binary: + left: + Literal: + Integer: 3 + span: "0:5-6" + op: Pow + right: + Literal: + Integer: 4 + span: "0:10-11" + span: "0:5-11" + span: "0:0-11" + "###); + assert_yaml_snapshot!(parse_expr(r#"1 + 2 ** (3 + 4) ** 4"#).unwrap(), @r###" + --- + Binary: + left: + Literal: + Integer: 1 + span: "0:0-1" + op: Add + right: + Binary: + left: + Literal: + Integer: 2 + span: "0:4-5" + op: Pow + right: + Binary: + left: + Binary: + left: + Literal: + Integer: 3 + span: "0:10-11" + op: Add + right: + Literal: + Integer: 4 + span: "0:14-15" + span: "0:10-15" + op: Pow + right: + Literal: + Integer: 4 + span: "0:20-21" + span: "0:9-21" + span: "0:4-21" + span: "0:0-21" + "###); +} + +#[test] +fn test_op_precedence() { + assert_yaml_snapshot!(parse_expr(r#"1 + 2 - 3 - 4"#).unwrap(), @r###" + --- + Binary: + left: + Binary: + left: + Binary: + left: + Literal: + Integer: 1 + span: "0:0-1" + op: Add + right: + Literal: + Integer: 2 + span: "0:4-5" + span: "0:0-5" + op: Sub + right: + Literal: + Integer: 3 + span: "0:8-9" + span: "0:0-9" + op: Sub + right: + Literal: + Integer: 4 + span: "0:12-13" + span: "0:0-13" + "###); + + assert_yaml_snapshot!(parse_expr(r#"1 / (3 * 4)"#).unwrap(), @r###" + --- + Binary: + left: + Literal: + Integer: 1 + span: "0:0-1" + op: DivFloat + right: + Binary: + left: + Literal: + Integer: 3 + span: "0:5-6" + op: Mul + right: + Literal: + Integer: 4 + span: "0:9-10" + span: "0:5-10" + span: "0:0-11" + "###); + + assert_yaml_snapshot!(parse_expr(r#"1 / 2 - 3 * 4 + 1"#).unwrap(), @r###" + --- + Binary: + left: + Binary: + left: + Binary: + left: + Literal: + Integer: 1 + span: "0:0-1" + op: DivFloat + right: + Literal: + Integer: 2 + span: "0:4-5" + span: "0:0-5" + op: Sub + right: + Binary: + left: + Literal: + Integer: 3 + span: "0:8-9" + op: Mul + right: + Literal: + Integer: 4 + span: "0:12-13" + span: "0:8-13" + span: "0:0-13" + op: Add + right: + Literal: + Integer: 1 + span: "0:16-17" + span: "0:0-17" + "###); + + assert_yaml_snapshot!(parse_expr(r#"a && b || !c && d"#).unwrap(), @r###" + --- + Binary: + left: + Binary: + left: + Ident: a + span: "0:0-1" + op: And + right: + Ident: b + span: "0:5-6" + span: "0:0-6" + op: Or + right: + Binary: + left: + Unary: + op: Not + expr: + Ident: c + span: "0:11-12" + span: "0:10-12" + op: And + right: + Ident: d + span: "0:16-17" + span: "0:10-17" + span: "0:0-17" + "###); + + assert_yaml_snapshot!(parse_expr(r#"a && b + c || (d e) && f"#).unwrap(), @r###" + --- + Binary: + left: + Binary: + left: + Ident: a + span: "0:0-1" + op: And + right: + Binary: + left: + Ident: b + span: "0:5-6" + op: Add + right: + Ident: c + span: "0:9-10" + span: "0:5-10" + span: "0:0-10" + op: Or + right: + Binary: + left: + FuncCall: + name: + Ident: d + span: "0:15-16" + args: + - Ident: e + span: "0:17-18" + span: "0:15-18" + op: And + right: + Ident: f + span: "0:23-24" + span: "0:14-24" + span: "0:0-24" + "###); +} + +#[test] +fn test_inline_pipeline() { + assert_yaml_snapshot!(parse_expr("(salary | percentile 50)").unwrap(), @r###" + --- + Pipeline: + exprs: + - Ident: salary + span: "0:1-7" + - FuncCall: + name: + Ident: percentile + span: "0:10-20" + args: + - Literal: + Integer: 50 + span: "0:21-23" + span: "0:10-23" + span: "0:0-24" + "###); +} + +#[test] +fn test_dates() { + assert_yaml_snapshot!(parse_expr("@2011-02-01").unwrap(), @r###" + --- + Literal: + Date: 2011-02-01 + span: "0:0-11" + "###); + assert_yaml_snapshot!(parse_expr("@2011-02-01T10:00").unwrap(), @r###" + --- + Literal: + Timestamp: "2011-02-01T10:00" + span: "0:0-17" + "###); + assert_yaml_snapshot!(parse_expr("@14:00").unwrap(), @r###" + --- + Literal: + Time: "14:00" + span: "0:0-6" + "###); + // assert_yaml_snapshot!(parse_expr("@2011-02-01T10:00").unwrap(), @""); + + parse_expr("@2020-01-0").unwrap_err(); + + parse_expr("@2020-01-011").unwrap_err(); + + parse_expr("@2020-01-01T111").unwrap_err(); +} + +#[test] +fn test_ident_with_keywords() { + assert_yaml_snapshot!(parse_expr(r"select {andrew, orion, lettuce, falsehood, null0}").unwrap(), @r###" + --- + FuncCall: + name: + Ident: select + span: "0:0-6" + args: + - Tuple: + - Ident: andrew + span: "0:8-14" + - Ident: orion + span: "0:16-21" + - Ident: lettuce + span: "0:23-30" + - Ident: falsehood + span: "0:32-41" + - Ident: null0 + span: "0:43-48" + span: "0:7-49" + span: "0:0-49" + "###); + + assert_yaml_snapshot!(parse_expr(r"{false}").unwrap(), @r###" + --- + Tuple: + - Literal: + Boolean: false + span: "0:1-6" + span: "0:0-7" + "###); +} + +#[test] +fn test_case() { + assert_yaml_snapshot!(parse_expr(r#"case [ + nickname != null => nickname, + true => null + ]"#).unwrap(), @r###" + --- + Case: + - condition: + Binary: + left: + Ident: nickname + span: "0:19-27" + op: Ne + right: + Literal: "Null" + span: "0:31-35" + span: "0:19-35" + value: + Ident: nickname + span: "0:39-47" + - condition: + Literal: + Boolean: true + span: "0:61-65" + value: + Literal: "Null" + span: "0:69-73" + span: "0:0-83" + "###); +} + +#[test] +fn test_params() { + assert_yaml_snapshot!(parse_expr(r#"$2"#).unwrap(), @r###" + --- + Param: "2" + span: "0:0-2" + "###); + + assert_yaml_snapshot!(parse_expr(r#"$2_any_text"#).unwrap(), @r###" + --- + Param: 2_any_text + span: "0:0-11" + "###); +} + +#[test] +fn test_lookup_01() { + assert_yaml_snapshot!(parse_expr( + r#"{a = {x = 2}}.a.x"#, + ).unwrap(), @r###" + --- + Indirection: + base: + Indirection: + base: + Tuple: + - Tuple: + - Literal: + Integer: 2 + span: "0:10-11" + alias: x + span: "0:5-12" + alias: a + span: "0:0-13" + field: + Name: a + span: "0:13-15" + field: + Name: x + span: "0:0-17" + "###); +} + +#[test] +fn test_lookup_02() { + assert_yaml_snapshot!(parse_expr( + r#"hello.*"#, + ).unwrap(), @r###" + --- + Indirection: + base: + Ident: hello + span: "0:0-5" + field: Star + span: "0:0-7" + "###); +} diff --git a/prqlc/prqlc-parser/src/test.rs b/prqlc/prqlc-parser/src/test.rs index 70ab97f1a175..65937607de0c 100644 --- a/prqlc/prqlc-parser/src/test.rs +++ b/prqlc/prqlc-parser/src/test.rs @@ -1,29 +1,32 @@ +use chumsky::Parser; use insta::{assert_debug_snapshot, assert_yaml_snapshot}; -use itertools::Itertools; -use crate::error::Error; -use crate::lexer; -use crate::parser; -use crate::parser::pr::{Expr, FuncCall, Stmt}; +use crate::parser::pr::Stmt; +use crate::parser::prepare_stream; +use crate::parser::stmt; +use crate::{error::Error, lexer::lr::TokenKind, parser::perror::PError}; -/// Helper that does not track source_ids -fn parse_single(source: &str) -> Result, Vec> { - let tokens = lexer::lex_source(source)?; +/// Parse source code based on the supplied parser. +/// +/// Use this to test any parser! +pub(crate) fn parse_with_parser( + source: &str, + parser: impl Parser, +) -> Result> { + let tokens = crate::lexer::lex_source(source)?; + let stream = prepare_stream(tokens.0.into_iter(), source, 0); - let (ast, parse_errors) = parser::parse_lr_to_pr(source, 0, tokens.0); + let (ast, parse_errors) = parser.parse_recovery(stream); if !parse_errors.is_empty() { - return Err(parse_errors); + return Err(parse_errors.into_iter().map(|e| e.into()).collect()); } - Ok(ast.unwrap_or_default()) + Ok(ast.unwrap()) } -fn parse_expr(source: &str) -> Result> { - let source = format!("let result = ({source}\n)"); - - let stmts = parse_single(&source)?; - let stmt = stmts.into_iter().exactly_one().unwrap(); - Ok(*stmt.kind.into_var_def().unwrap().value.unwrap()) +/// Parse into statements +fn parse_single(source: &str) -> Result, Vec> { + parse_with_parser(source, stmt::source()) } #[test] @@ -185,674 +188,6 @@ fn test_take() { "###); } -#[test] -fn test_ranges() { - assert_yaml_snapshot!(parse_expr(r#"3..5"#).unwrap(), @r###" - --- - Range: - start: - Literal: - Integer: 3 - span: "0:14-15" - end: - Literal: - Integer: 5 - span: "0:17-18" - span: "0:13-20" - "###); - - assert_yaml_snapshot!(parse_expr(r#"-2..-5"#).unwrap(), @r###" - --- - Range: - start: - Unary: - op: Neg - expr: - Literal: - Integer: 2 - span: "0:15-16" - span: "0:14-16" - end: - Unary: - op: Neg - expr: - Literal: - Integer: 5 - span: "0:19-20" - span: "0:18-20" - span: "0:13-22" - "###); - - assert_yaml_snapshot!(parse_expr(r#"(-2..(-5 | abs))"#).unwrap(), @r###" - --- - Range: - start: - Unary: - op: Neg - expr: - Literal: - Integer: 2 - span: "0:16-17" - span: "0:15-17" - end: - Pipeline: - exprs: - - Unary: - op: Neg - expr: - Literal: - Integer: 5 - span: "0:21-22" - span: "0:20-22" - - Ident: abs - span: "0:25-28" - span: "0:20-28" - span: "0:13-32" - "###); - - assert_yaml_snapshot!(parse_expr(r#"(2 + 5)..'a'"#).unwrap(), @r###" - --- - Range: - start: - Binary: - left: - Literal: - Integer: 2 - span: "0:15-16" - op: Add - right: - Literal: - Integer: 5 - span: "0:19-20" - span: "0:15-20" - end: - Literal: - String: a - span: "0:23-26" - span: "0:13-28" - "###); - - assert_yaml_snapshot!(parse_expr(r#"1.6..rel.col"#).unwrap(), @r###" - --- - Range: - start: - Literal: - Float: 1.6 - span: "0:14-17" - end: - Indirection: - base: - Ident: rel - span: "0:19-22" - field: - Name: col - span: "0:22-26" - span: "0:13-28" - "###); - - assert_yaml_snapshot!(parse_expr(r#"6.."#).unwrap(), @r###" - --- - Range: - start: - Literal: - Integer: 6 - span: "0:14-15" - end: ~ - span: "0:13-19" - "###); - assert_yaml_snapshot!(parse_expr(r#"..7"#).unwrap(), @r###" - --- - Range: - start: ~ - end: - Literal: - Integer: 7 - span: "0:16-17" - span: "0:13-19" - "###); - - assert_yaml_snapshot!(parse_expr(r#".."#).unwrap(), @r###" - --- - Range: - start: ~ - end: ~ - span: "0:13-18" - "###); - - assert_yaml_snapshot!(parse_expr(r#"@2020-01-01..@2021-01-01"#).unwrap(), @r###" - --- - Range: - start: - Literal: - Date: 2020-01-01 - span: "0:14-25" - end: - Literal: - Date: 2021-01-01 - span: "0:27-38" - span: "0:13-40" - "###); -} - -#[test] -fn test_basic_exprs() { - assert_yaml_snapshot!(parse_expr(r#"country == "USA""#).unwrap(), @r###" - --- - Binary: - left: - Ident: country - span: "0:14-21" - op: Eq - right: - Literal: - String: USA - span: "0:25-30" - span: "0:13-32" - "###); - assert_yaml_snapshot!(parse_expr("select {a, b, c}").unwrap(), @r###" - --- - FuncCall: - name: - Ident: select - span: "0:14-20" - args: - - Tuple: - - Ident: a - span: "0:22-23" - - Ident: b - span: "0:25-26" - - Ident: c - span: "0:28-29" - span: "0:21-30" - span: "0:13-32" - "###); - assert_yaml_snapshot!(parse_expr( - "group {title, country} ( - aggregate {sum salary} - )" - ).unwrap(), @r###" - --- - FuncCall: - name: - Ident: group - span: "0:14-19" - args: - - Tuple: - - Ident: title - span: "0:21-26" - - Ident: country - span: "0:28-35" - span: "0:20-36" - - FuncCall: - name: - Ident: aggregate - span: "0:55-64" - args: - - Tuple: - - FuncCall: - name: - Ident: sum - span: "0:66-69" - args: - - Ident: salary - span: "0:70-76" - span: "0:66-76" - span: "0:65-77" - span: "0:55-77" - span: "0:13-93" - "###); - assert_yaml_snapshot!(parse_expr( - r#" filter country == "USA""# - ).unwrap(), @r###" - --- - FuncCall: - name: - Ident: filter - span: "0:18-24" - args: - - Binary: - left: - Ident: country - span: "0:25-32" - op: Eq - right: - Literal: - String: USA - span: "0:36-41" - span: "0:25-41" - span: "0:13-43" - "###); - assert_yaml_snapshot!(parse_expr("{a, b, c,}").unwrap(), @r###" - --- - Tuple: - - Ident: a - span: "0:15-16" - - Ident: b - span: "0:18-19" - - Ident: c - span: "0:21-22" - span: "0:13-26" - "###); - assert_yaml_snapshot!(parse_expr( - r#"{ - gross_salary = salary + payroll_tax, - gross_cost = gross_salary + benefits_cost -}"# - ).unwrap(), @r###" - --- - Tuple: - - Binary: - left: - Ident: salary - span: "0:33-39" - op: Add - right: - Ident: payroll_tax - span: "0:42-53" - span: "0:33-53" - alias: gross_salary - - Binary: - left: - Ident: gross_salary - span: "0:72-84" - op: Add - right: - Ident: benefits_cost - span: "0:87-100" - span: "0:72-100" - alias: gross_cost - span: "0:13-104" - "###); - // Currently not putting comments in our parse tree, so this is blank. - assert_yaml_snapshot!(parse_single( - r#"# this is a comment - select a"# - ).unwrap(), @r###" - --- - - VarDef: - kind: Main - name: main - value: - FuncCall: - name: - Ident: select - span: "0:28-34" - args: - - Ident: a - span: "0:35-36" - span: "0:28-36" - span: "0:28-36" - "###); - assert_yaml_snapshot!(parse_expr( - "join side:left country (id==employee_id)" - ).unwrap(), @r###" - --- - FuncCall: - name: - Ident: join - span: "0:14-18" - args: - - Ident: country - span: "0:29-36" - - Binary: - left: - Ident: id - span: "0:38-40" - op: Eq - right: - Ident: employee_id - span: "0:42-53" - span: "0:38-53" - named_args: - side: - Ident: left - span: "0:24-28" - span: "0:13-56" - "###); - assert_yaml_snapshot!(parse_expr("1 + 2").unwrap(), @r###" - --- - Binary: - left: - Literal: - Integer: 1 - span: "0:14-15" - op: Add - right: - Literal: - Integer: 2 - span: "0:19-20" - span: "0:13-22" - "###) -} - -#[test] -fn test_string() { - let double_quoted_ast = parse_expr(r#"" U S A ""#).unwrap(); - assert_yaml_snapshot!(double_quoted_ast, @r###" - --- - Literal: - String: " U S A " - span: "0:13-25" - "###); - - let single_quoted_ast = parse_expr(r#"' U S A '"#).unwrap(); - assert_eq!(single_quoted_ast, double_quoted_ast); - - // Single quotes within double quotes should produce a string containing - // the single quotes (and vice versa). - assert_yaml_snapshot!(parse_expr(r#""' U S A '""#).unwrap(), @r###" - --- - Literal: - String: "' U S A '" - span: "0:13-27" - "###); - assert_yaml_snapshot!(parse_expr(r#"'" U S A "'"#).unwrap(), @r###" - --- - Literal: - String: "\" U S A \"" - span: "0:13-27" - "###); - - parse_expr(r#"" U S A"#).unwrap_err(); - parse_expr(r#"" U S A '"#).unwrap_err(); - - assert_yaml_snapshot!(parse_expr(r#"" \nU S A ""#).unwrap(), @r###" - --- - Literal: - String: " \nU S A " - span: "0:13-27" - "###); - - assert_yaml_snapshot!(parse_expr(r#"r" \nU S A ""#).unwrap(), @r###" - --- - Literal: - String: " \\nU S A " - span: "0:13-28" - "###); - - let multi_double = parse_expr( - r#"""" -'' -Canada -" - -""""#, - ) - .unwrap(); - assert_yaml_snapshot!(multi_double, @r###" - --- - Literal: - String: "\n''\nCanada\n\"\n\n" - span: "0:13-36" - "###); - - let multi_single = parse_expr( - r#"''' -Canada -" -""" - -'''"#, - ) - .unwrap(); - assert_yaml_snapshot!(multi_single, @r###" - --- - Literal: - String: "\nCanada\n\"\n\"\"\"\n\n" - span: "0:13-37" - "###); - - assert_yaml_snapshot!( - parse_expr("''").unwrap(), - @r###" - --- - Literal: - String: "" - span: "0:13-18" - "###); -} - -#[test] -fn test_s_string() { - assert_yaml_snapshot!(parse_expr(r#"s"SUM({col})""#).unwrap(), @r###" - --- - SString: - - String: SUM( - - Expr: - expr: - Ident: col - span: "0:21-24" - format: ~ - - String: ) - span: "0:13-29" - "###); - assert_yaml_snapshot!(parse_expr(r#"s"SUM({rel.`Col name`})""#).unwrap(), @r###" - --- - SString: - - String: SUM( - - Expr: - expr: - Indirection: - base: - Ident: rel - span: "0:21-24" - field: - Name: Col name - span: "0:25-35" - format: ~ - - String: ) - span: "0:13-40" - "###) -} - -#[test] -fn test_s_string_braces() { - assert_yaml_snapshot!(parse_expr(r#"s"{{?crystal_var}}""#).unwrap(), @r###" - --- - SString: - - String: "{?crystal_var}" - span: "0:13-35" - "###); - assert_yaml_snapshot!(parse_expr(r#"s"foo{{bar""#).unwrap(), @r###" - --- - SString: - - String: "foo{bar" - span: "0:13-27" - "###); - parse_expr(r#"s"foo{{bar}""#).unwrap_err(); -} - -#[test] -fn test_tuple() { - assert_yaml_snapshot!(parse_expr(r#"{1 + 1, 2}"#).unwrap(), @r###" - --- - Tuple: - - Binary: - left: - Literal: - Integer: 1 - span: "0:15-16" - op: Add - right: - Literal: - Integer: 1 - span: "0:19-20" - span: "0:15-20" - - Literal: - Integer: 2 - span: "0:22-23" - span: "0:13-26" - "###); - assert_yaml_snapshot!(parse_expr(r#"{1 + (f 1), 2}"#).unwrap(), @r###" - --- - Tuple: - - Binary: - left: - Literal: - Integer: 1 - span: "0:15-16" - op: Add - right: - FuncCall: - name: - Ident: f - span: "0:20-21" - args: - - Literal: - Integer: 1 - span: "0:22-23" - span: "0:20-23" - span: "0:15-24" - - Literal: - Integer: 2 - span: "0:26-27" - span: "0:13-30" - "###); - // Line breaks - assert_yaml_snapshot!(parse_expr( - r#"{1, - - 2}"# - ).unwrap(), @r###" - --- - Tuple: - - Literal: - Integer: 1 - span: "0:15-16" - - Literal: - Integer: 2 - span: "0:35-36" - span: "0:13-39" - "###); - // Function call in a tuple - let ab = parse_expr(r#"{a b}"#).unwrap(); - let a_comma_b = parse_expr(r#"{a, b}"#).unwrap(); - assert_yaml_snapshot!(ab, @r###" - --- - Tuple: - - FuncCall: - name: - Ident: a - span: "0:15-16" - args: - - Ident: b - span: "0:17-18" - span: "0:15-18" - span: "0:13-21" - "###); - assert_yaml_snapshot!(a_comma_b, @r###" - --- - Tuple: - - Ident: a - span: "0:15-16" - - Ident: b - span: "0:18-19" - span: "0:13-22" - "###); - assert_ne!(ab, a_comma_b); - - assert_yaml_snapshot!(parse_expr(r#"{amount, +amount, -amount}"#).unwrap(), @r###" - --- - Tuple: - - Ident: amount - span: "0:15-21" - - Unary: - op: Add - expr: - Ident: amount - span: "0:24-30" - span: "0:23-30" - - Unary: - op: Neg - expr: - Ident: amount - span: "0:33-39" - span: "0:32-39" - span: "0:13-42" - "###); - // Operators in tuple items - assert_yaml_snapshot!(parse_expr(r#"{amount, +amount, -amount}"#).unwrap(), @r###" - --- - Tuple: - - Ident: amount - span: "0:15-21" - - Unary: - op: Add - expr: - Ident: amount - span: "0:24-30" - span: "0:23-30" - - Unary: - op: Neg - expr: - Ident: amount - span: "0:33-39" - span: "0:32-39" - span: "0:13-42" - "###); -} - -#[test] -fn test_number() { - assert_yaml_snapshot!(parse_expr(r#"23"#).unwrap(), @r###" - --- - Literal: - Integer: 23 - span: "0:13-18" - "###); - assert_yaml_snapshot!(parse_expr(r#"2_3_4.5_6"#).unwrap(), @r###" - --- - Literal: - Float: 234.56 - span: "0:13-25" - "###); - assert_yaml_snapshot!(parse_expr(r#"23.6"#).unwrap(), @r###" - --- - Literal: - Float: 23.6 - span: "0:13-20" - "###); - assert_yaml_snapshot!(parse_expr(r#"23.0"#).unwrap(), @r###" - --- - Literal: - Float: 23 - span: "0:13-20" - "###); - assert_yaml_snapshot!(parse_expr(r#"2 + 2"#).unwrap(), @r###" - --- - Binary: - left: - Literal: - Integer: 2 - span: "0:14-15" - op: Add - right: - Literal: - Integer: 2 - span: "0:18-19" - span: "0:13-21" - "###); - - // Underscores at the beginning are parsed as ident - assert!(parse_expr("_2").unwrap().kind.into_ident().is_ok()); - assert!(parse_expr("_").unwrap().kind.into_ident().is_ok()); - - // We don't allow trailing periods - assert!(parse_expr(r#"add 1. (2, 3)"#).is_err()); - - assert!(parse_expr("_2.3").unwrap().kind.is_indirection()); - - assert_yaml_snapshot!(parse_expr(r#"2e3"#).unwrap(), @r###" - --- - Literal: - Float: 2000 - span: "0:13-19" - "###); - - // expr_of_string("2_").unwrap_err(); // TODO - // expr_of_string("2.3_").unwrap_err(); // TODO -} - #[test] fn test_filter() { assert_yaml_snapshot!( @@ -1009,186 +344,26 @@ fn test_aggregate() { } #[test] -fn test_derive() { - assert_yaml_snapshot!( - parse_expr(r#"derive {x = 5, y = (-x)}"#).unwrap() - , @r###" - --- - FuncCall: - name: - Ident: derive - span: "0:14-20" - args: - - Tuple: - - Literal: - Integer: 5 - span: "0:26-27" - alias: x - - Unary: - op: Neg - expr: - Ident: x - span: "0:35-36" - span: "0:33-37" - alias: y - span: "0:21-38" - span: "0:13-40" - "###); -} - -#[test] -fn test_select() { - assert_yaml_snapshot!( - parse_expr(r#"select x"#).unwrap() - , @r###" - --- - FuncCall: - name: - Ident: select - span: "0:14-20" - args: - - Ident: x - span: "0:21-22" - span: "0:13-24" - "###); - - assert_yaml_snapshot!( - parse_expr(r#"select !{x}"#).unwrap() - , @r###" - --- - FuncCall: - name: - Ident: select - span: "0:14-20" - args: - - Unary: - op: Not - expr: - Tuple: - - Ident: x - span: "0:23-24" - span: "0:22-25" - span: "0:21-25" - span: "0:13-27" - "###); - - assert_yaml_snapshot!( - parse_expr(r#"select {x, y}"#).unwrap() - , @r###" - --- - FuncCall: - name: - Ident: select - span: "0:14-20" - args: - - Tuple: - - Ident: x - span: "0:22-23" - - Ident: y - span: "0:25-26" - span: "0:21-27" - span: "0:13-29" - "###); -} - -#[test] -fn test_expr() { - assert_yaml_snapshot!( - parse_expr(r#"country == "USA""#).unwrap() - , @r###" - --- - Binary: - left: - Ident: country - span: "0:14-21" - op: Eq - right: - Literal: - String: USA - span: "0:25-30" - span: "0:13-32" - "###); - assert_yaml_snapshot!(parse_expr( - r#"{ - gross_salary = salary + payroll_tax, - gross_cost = gross_salary + benefits_cost, -}"#).unwrap(), @r###" - --- - Tuple: - - Binary: - left: - Ident: salary - span: "0:33-39" - op: Add - right: - Ident: payroll_tax - span: "0:42-53" - span: "0:33-53" - alias: gross_salary - - Binary: - left: - Ident: gross_salary - span: "0:72-84" - op: Add - right: - Ident: benefits_cost - span: "0:87-100" - span: "0:72-100" - alias: gross_cost - span: "0:13-105" - "###); - assert_yaml_snapshot!( - parse_expr( - "(salary + payroll_tax) * (1 + tax_rate)" - ).unwrap(), - @r###" - --- - Binary: - left: - Binary: - left: - Ident: salary - span: "0:15-21" - op: Add - right: - Ident: payroll_tax - span: "0:24-35" - span: "0:15-35" - op: Mul - right: - Binary: - left: - Literal: - Integer: 1 - span: "0:40-41" - op: Add - right: - Ident: tax_rate - span: "0:44-52" - span: "0:40-52" - span: "0:13-55" - "###); -} - -#[test] -fn test_regex() { - assert_yaml_snapshot!( - parse_expr( - "'oba' ~= 'foobar'" - ).unwrap(), - @r###" +fn test_basic_exprs() { + // Currently not putting comments in our parse tree, so this is blank. + assert_yaml_snapshot!(parse_single( + r#"# this is a comment + select a"# + ).unwrap(), @r###" --- - Binary: - left: - Literal: - String: oba - span: "0:14-19" - op: RegexSearch - right: - Literal: - String: foobar - span: "0:23-31" - span: "0:13-33" + - VarDef: + kind: Main + name: main + value: + FuncCall: + name: + Ident: select + span: "0:28-34" + args: + - Ident: a + span: "0:35-36" + span: "0:28-36" + span: "0:28-36" "###); } @@ -1493,299 +668,6 @@ fn test_function() { "###); } -#[test] -fn test_func_call() { - // Function without argument - let ast = parse_expr(r#"count"#).unwrap(); - let ident = ast.kind.into_ident().unwrap(); - assert_yaml_snapshot!( - ident, @r###" - --- - count - "###); - - let ast = parse_expr(r#"s 'foo'"#).unwrap(); - assert_yaml_snapshot!( - ast, @r###" - --- - FuncCall: - name: - Ident: s - span: "0:14-15" - args: - - Literal: - String: foo - span: "0:16-21" - span: "0:13-23" - "###); - - // A non-friendly option for #154 - let ast = parse_expr(r#"count s'*'"#).unwrap(); - let func_call: FuncCall = ast.kind.into_func_call().unwrap(); - assert_yaml_snapshot!( - func_call, @r###" - --- - name: - Ident: count - span: "0:14-19" - args: - - SString: - - String: "*" - span: "0:20-24" - "###); - - parse_expr("plus_one x:0 x:0 ").unwrap_err(); - - let ast = parse_expr(r#"add bar to=3"#).unwrap(); - assert_yaml_snapshot!( - ast, @r###" - --- - FuncCall: - name: - Ident: add - span: "0:14-17" - args: - - Ident: bar - span: "0:18-21" - - Literal: - Integer: 3 - span: "0:25-26" - alias: to - span: "0:13-28" - "###); -} - -#[test] -fn test_right_assoc() { - assert_yaml_snapshot!(parse_expr(r#"2 ** 3 ** 4"#).unwrap(), @r###" - --- - Binary: - left: - Literal: - Integer: 2 - span: "0:14-15" - op: Pow - right: - Binary: - left: - Literal: - Integer: 3 - span: "0:19-20" - op: Pow - right: - Literal: - Integer: 4 - span: "0:24-25" - span: "0:19-25" - span: "0:13-27" - "###); - assert_yaml_snapshot!(parse_expr(r#"1 + 2 ** (3 + 4) ** 4"#).unwrap(), @r###" - --- - Binary: - left: - Literal: - Integer: 1 - span: "0:14-15" - op: Add - right: - Binary: - left: - Literal: - Integer: 2 - span: "0:18-19" - op: Pow - right: - Binary: - left: - Binary: - left: - Literal: - Integer: 3 - span: "0:24-25" - op: Add - right: - Literal: - Integer: 4 - span: "0:28-29" - span: "0:24-29" - op: Pow - right: - Literal: - Integer: 4 - span: "0:34-35" - span: "0:23-35" - span: "0:18-35" - span: "0:13-37" - "###); -} - -#[test] -fn test_op_precedence() { - assert_yaml_snapshot!(parse_expr(r#"1 + 2 - 3 - 4"#).unwrap(), @r###" - --- - Binary: - left: - Binary: - left: - Binary: - left: - Literal: - Integer: 1 - span: "0:14-15" - op: Add - right: - Literal: - Integer: 2 - span: "0:18-19" - span: "0:14-19" - op: Sub - right: - Literal: - Integer: 3 - span: "0:22-23" - span: "0:14-23" - op: Sub - right: - Literal: - Integer: 4 - span: "0:26-27" - span: "0:13-29" - "###); - - assert_yaml_snapshot!(parse_expr(r#"1 / (3 * 4)"#).unwrap(), @r###" - --- - Binary: - left: - Literal: - Integer: 1 - span: "0:14-15" - op: DivFloat - right: - Binary: - left: - Literal: - Integer: 3 - span: "0:19-20" - op: Mul - right: - Literal: - Integer: 4 - span: "0:23-24" - span: "0:19-24" - span: "0:13-27" - "###); - - assert_yaml_snapshot!(parse_expr(r#"1 / 2 - 3 * 4 + 1"#).unwrap(), @r###" - --- - Binary: - left: - Binary: - left: - Binary: - left: - Literal: - Integer: 1 - span: "0:14-15" - op: DivFloat - right: - Literal: - Integer: 2 - span: "0:18-19" - span: "0:14-19" - op: Sub - right: - Binary: - left: - Literal: - Integer: 3 - span: "0:22-23" - op: Mul - right: - Literal: - Integer: 4 - span: "0:26-27" - span: "0:22-27" - span: "0:14-27" - op: Add - right: - Literal: - Integer: 1 - span: "0:30-31" - span: "0:13-33" - "###); - - assert_yaml_snapshot!(parse_expr(r#"a && b || !c && d"#).unwrap(), @r###" - --- - Binary: - left: - Binary: - left: - Ident: a - span: "0:14-15" - op: And - right: - Ident: b - span: "0:19-20" - span: "0:14-20" - op: Or - right: - Binary: - left: - Unary: - op: Not - expr: - Ident: c - span: "0:25-26" - span: "0:24-26" - op: And - right: - Ident: d - span: "0:30-31" - span: "0:24-31" - span: "0:13-33" - "###); - - assert_yaml_snapshot!(parse_expr(r#"a && b + c || (d e) && f"#).unwrap(), @r###" - --- - Binary: - left: - Binary: - left: - Ident: a - span: "0:14-15" - op: And - right: - Binary: - left: - Ident: b - span: "0:19-20" - op: Add - right: - Ident: c - span: "0:23-24" - span: "0:19-24" - span: "0:14-24" - op: Or - right: - Binary: - left: - FuncCall: - name: - Ident: d - span: "0:29-30" - args: - - Ident: e - span: "0:31-32" - span: "0:29-32" - op: And - right: - Ident: f - span: "0:37-38" - span: "0:28-38" - span: "0:13-40" - "###); -} - #[test] fn test_var_def() { assert_yaml_snapshot!(parse_single( @@ -1950,23 +832,6 @@ fn test_var_def() { #[test] fn test_inline_pipeline() { - assert_yaml_snapshot!(parse_expr("(salary | percentile 50)").unwrap(), @r###" - --- - Pipeline: - exprs: - - Ident: salary - span: "0:15-21" - - FuncCall: - name: - Ident: percentile - span: "0:24-34" - args: - - Literal: - Integer: 50 - span: "0:35-37" - span: "0:24-37" - span: "0:13-40" - "###); assert_yaml_snapshot!(parse_single("let median = x -> (x | percentile 50)\n").unwrap(), @r###" --- - VarDef: @@ -2297,32 +1162,6 @@ fn test_dates() { span: "0:9-76" span: "0:9-77" "###); - - assert_yaml_snapshot!(parse_expr("@2011-02-01").unwrap(), @r###" - --- - Literal: - Date: 2011-02-01 - span: "0:13-27" - "###); - assert_yaml_snapshot!(parse_expr("@2011-02-01T10:00").unwrap(), @r###" - --- - Literal: - Timestamp: "2011-02-01T10:00" - span: "0:13-33" - "###); - assert_yaml_snapshot!(parse_expr("@14:00").unwrap(), @r###" - --- - Literal: - Time: "14:00" - span: "0:13-22" - "###); - // assert_yaml_snapshot!(parse_expr("@2011-02-01T10:00").unwrap(), @""); - - parse_expr("@2020-01-0").unwrap_err(); - - parse_expr("@2020-01-011").unwrap_err(); - - parse_expr("@2020-01-01T111").unwrap_err(); } #[test] @@ -2663,87 +1502,6 @@ join s=salaries (==id) "###); } -#[test] -fn test_ident_with_keywords() { - assert_yaml_snapshot!(parse_expr(r"select {andrew, orion, lettuce, falsehood, null0}").unwrap(), @r###" - --- - FuncCall: - name: - Ident: select - span: "0:14-20" - args: - - Tuple: - - Ident: andrew - span: "0:22-28" - - Ident: orion - span: "0:30-35" - - Ident: lettuce - span: "0:37-44" - - Ident: falsehood - span: "0:46-55" - - Ident: null0 - span: "0:57-62" - span: "0:21-63" - span: "0:13-65" - "###); - - assert_yaml_snapshot!(parse_expr(r"{false}").unwrap(), @r###" - --- - Tuple: - - Literal: - Boolean: false - span: "0:15-20" - span: "0:13-23" - "###); -} - -#[test] -fn test_case() { - assert_yaml_snapshot!(parse_expr(r#"case [ - nickname != null => nickname, - true => null - ]"#).unwrap(), @r###" - --- - Case: - - condition: - Binary: - left: - Ident: nickname - span: "0:33-41" - op: Ne - right: - Literal: "Null" - span: "0:45-49" - span: "0:33-49" - value: - Ident: nickname - span: "0:53-61" - - condition: - Literal: - Boolean: true - span: "0:75-79" - value: - Literal: "Null" - span: "0:83-87" - span: "0:13-99" - "###); -} - -#[test] -fn test_params() { - assert_yaml_snapshot!(parse_expr(r#"$2"#).unwrap(), @r###" - --- - Param: "2" - span: "0:13-18" - "###); - - assert_yaml_snapshot!(parse_expr(r#"$2_any_text"#).unwrap(), @r###" - --- - Param: 2_any_text - span: "0:13-27" - "###); -} - #[test] fn test_unicode() { let source = "from tète"; @@ -3037,48 +1795,12 @@ fn test_module() { } #[test] -fn test_lookup_01() { - assert_yaml_snapshot!(parse_expr( - r#" - {a = {x = 2}}.a.x - "#, - ).unwrap(), @r###" - --- - Indirection: - base: - Indirection: - base: - Tuple: - - Tuple: - - Literal: - Integer: 2 - span: "0:31-32" - alias: x - span: "0:26-33" - alias: a - span: "0:21-34" - field: - Name: a - span: "0:34-36" - field: - Name: x - span: "0:13-45" - "###); -} - -#[test] -fn test_lookup_02() { - assert_yaml_snapshot!(parse_expr( - r#" - hello.* - "#, - ).unwrap(), @r###" - --- - Indirection: - base: - Ident: hello - span: "0:21-26" - field: Star - span: "0:13-35" - "###); +fn test_number() { + // We don't allow trailing periods + assert!(parse_single( + r#" + from artists + derive x = 1."# + ) + .is_err()); } From f70d8e37026e5b97476f9073db64fa8428319123 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Thu, 11 Jul 2024 12:44:19 -0700 Subject: [PATCH 13/27] internal: Add a `Start` token at the beginning of LR output (#4721) --- prqlc/prqlc-parser/src/lexer/lr.rs | 5 +++ prqlc/prqlc-parser/src/lexer/mod.rs | 33 ++++++++++++++++--- prqlc/prqlc-parser/src/lexer/test.rs | 1 + prqlc/prqlc-parser/src/parser/mod.rs | 5 ++- prqlc/prqlc/src/cli/mod.rs | 8 +++++ prqlc/prqlc/tests/integration/cli.rs | 11 +++++++ ...ntegration__queries__lex__aggregation.snap | 1 + ...integration__queries__lex__arithmetic.snap | 1 + .../integration__queries__lex__cast.snap | 1 + ...gration__queries__lex__constants_only.snap | 1 + ...tegration__queries__lex__date_to_text.snap | 1 + .../integration__queries__lex__distinct.snap | 1 + ...ntegration__queries__lex__distinct_on.snap | 1 + ...tegration__queries__lex__genre_counts.snap | 1 + .../integration__queries__lex__group_all.snap | 1 + ...integration__queries__lex__group_sort.snap | 1 + ...__queries__lex__group_sort_limit_take.snap | 1 + ...gration__queries__lex__invoice_totals.snap | 1 + .../integration__queries__lex__loop_01.snap | 1 + ...ntegration__queries__lex__math_module.snap | 1 + .../integration__queries__lex__pipelines.snap | 1 + .../integration__queries__lex__read_csv.snap | 1 + ...gration__queries__lex__set_ops_remove.snap | 1 + .../integration__queries__lex__sort.snap | 1 + .../integration__queries__lex__switch.snap | 1 + .../integration__queries__lex__take.snap | 1 + ...ntegration__queries__lex__text_module.snap | 1 + .../integration__queries__lex__window.snap | 1 + 28 files changed, 79 insertions(+), 6 deletions(-) diff --git a/prqlc/prqlc-parser/src/lexer/lr.rs b/prqlc/prqlc-parser/src/lexer/lr.rs index 1fa9e968d1a0..60a9ebb8c727 100644 --- a/prqlc/prqlc-parser/src/lexer/lr.rs +++ b/prqlc/prqlc-parser/src/lexer/lr.rs @@ -68,6 +68,10 @@ pub enum TokenKind { // - Change the functionality. But it's very nice to be able to comment // something out and have line-wraps still work. LineWrap(Vec), + + /// A token we manually insert at the start of the input, which later stages + /// can treat as a newline. + Start, } #[derive( @@ -229,6 +233,7 @@ impl std::fmt::Display for TokenKind { } Ok(()) } + TokenKind::Start => write!(f, "start of input"), } } } diff --git a/prqlc/prqlc-parser/src/lexer/mod.rs b/prqlc/prqlc-parser/src/lexer/mod.rs index cc3f07e474f2..f86cb319f1fa 100644 --- a/prqlc/prqlc-parser/src/lexer/mod.rs +++ b/prqlc/prqlc-parser/src/lexer/mod.rs @@ -10,8 +10,16 @@ pub mod lr; #[cfg(test)] mod test; +// TODO: we have `lex_source` and `lex_source_recovery` and don't have the same +// structure for PR. Probably we should have a single approach to the inclusion +// and naming of a function which returns both the tokens & errors, and a +// function that returns both. + +/// Lex PRQL into LR, returning both the LR and any errors encountered pub fn lex_source_recovery(source: &str, source_id: u16) -> (Option>, Vec) { - let (tokens, lex_errors) = ::chumsky::Parser::parse_recovery(&lexer(), source); + let (tokens, lex_errors) = lexer().parse_recovery(source); + + let tokens = tokens.map(insert_start); let errors = lex_errors .into_iter() @@ -22,12 +30,27 @@ pub fn lex_source_recovery(source: &str, source_id: u16) -> (Option>, (tokens, errors) } +/// Lex PRQL into LR, returning either the LR or the errors encountered pub fn lex_source(source: &str) -> Result> { - lexer().parse(source).map(lr::Tokens).map_err(|e| { - e.into_iter() - .map(|x| convert_lexer_error(source, x, 0)) - .collect() + lexer() + .parse(source) + .map(insert_start) + .map(lr::Tokens) + .map_err(|e| { + e.into_iter() + .map(|x| convert_lexer_error(source, x, 0)) + .collect() + }) +} + +/// Insert a start token so later stages can treat the start of a file like a newline +fn insert_start(tokens: Vec) -> Vec { + std::iter::once(Token { + kind: TokenKind::Start, + span: 0..0, }) + .chain(tokens) + .collect() } fn convert_lexer_error(source: &str, e: chumsky::error::Cheap, source_id: u16) -> Error { diff --git a/prqlc/prqlc-parser/src/lexer/test.rs b/prqlc/prqlc-parser/src/lexer/test.rs index 43249a45cd77..3dfdbe352049 100644 --- a/prqlc/prqlc-parser/src/lexer/test.rs +++ b/prqlc/prqlc-parser/src/lexer/test.rs @@ -197,6 +197,7 @@ fn test_lex_source() { Ok( Tokens( [ + 0..0: Start, 0..1: Literal(Integer(5)), 2..3: Control('+'), 4..5: Literal(Integer(3)), diff --git a/prqlc/prqlc-parser/src/parser/mod.rs b/prqlc/prqlc-parser/src/parser/mod.rs index 0e4c2798a3b3..d02f5985e90c 100644 --- a/prqlc/prqlc-parser/src/parser/mod.rs +++ b/prqlc/prqlc-parser/src/parser/mod.rs @@ -42,7 +42,10 @@ pub(crate) fn prepare_stream( let semantic_tokens = tokens.filter(|token| { !matches!( token.kind, - lr::TokenKind::Comment(_) | lr::TokenKind::LineWrap(_) | lr::TokenKind::DocComment(_) + lr::TokenKind::Comment(_) + | lr::TokenKind::LineWrap(_) + | lr::TokenKind::DocComment(_) + | lr::TokenKind::Start ) }); diff --git a/prqlc/prqlc/src/cli/mod.rs b/prqlc/prqlc/src/cli/mod.rs index 55bd42c3af3b..210102817ecb 100644 --- a/prqlc/prqlc/src/cli/mod.rs +++ b/prqlc/prqlc/src/cli/mod.rs @@ -775,6 +775,10 @@ sort full // TODO: terser output; maybe serialize span as `0..4`? Remove the // `!Ident` complication? assert_snapshot!(String::from_utf8(output).unwrap().trim(), @r###" + - kind: Start + span: + start: 0 + end: 0 - kind: !Ident from span: start: 0 @@ -814,6 +818,10 @@ sort full .unwrap(); assert_snapshot!(String::from_utf8(output).unwrap().trim(), @r###" + - kind: Start + span: + start: 0 + end: 0 - kind: NewLine span: start: 0 diff --git a/prqlc/prqlc/tests/integration/cli.rs b/prqlc/prqlc/tests/integration/cli.rs index 1cd2c0ecec12..ae39dc933b95 100644 --- a/prqlc/prqlc/tests/integration/cli.rs +++ b/prqlc/prqlc/tests/integration/cli.rs @@ -615,6 +615,10 @@ fn lex() { success: true exit_code: 0 ----- stdout ----- + - kind: Start + span: + start: 0 + end: 0 - kind: !Ident from span: start: 0 @@ -632,6 +636,13 @@ fn lex() { exit_code: 0 ----- stdout ----- [ + { + "kind": "Start", + "span": { + "start": 0, + "end": 0 + } + }, { "kind": { "Ident": "from" diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__aggregation.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__aggregation.snap index 96d0d765d3b6..51e6a0a93001 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__aggregation.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__aggregation.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/aggregation.prql --- Tokens( [ + 0..0: Start, 0..12: Comment(" mssql:skip"), 12..13: NewLine, 13..25: Comment(" mysql:skip"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__arithmetic.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__arithmetic.snap index d1577eeed4d1..be94046d2c4a 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__arithmetic.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__arithmetic.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/arithmetic.prql --- Tokens( [ + 0..0: Start, 0..12: Comment(" mssql:test"), 12..13: NewLine, 13..17: Ident("from"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__cast.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__cast.snap index 5ca3b24df716..078f09ca04ca 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__cast.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__cast.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/cast.prql --- Tokens( [ + 0..0: Start, 0..12: Comment(" mssql:test"), 12..13: NewLine, 13..17: Ident("from"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__constants_only.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__constants_only.snap index c38aeff36118..afb9d1de93bc 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__constants_only.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__constants_only.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/constants_only.prql --- Tokens( [ + 0..0: Start, 0..4: Ident("from"), 5..11: Ident("genres"), 11..12: NewLine, diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__date_to_text.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__date_to_text.snap index f70976159978..ab3ce3b0b905 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__date_to_text.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__date_to_text.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/date_to_text.prql --- Tokens( [ + 0..0: Start, 0..14: Comment(" generic:skip"), 14..15: NewLine, 15..29: Comment(" glaredb:skip"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct.snap index e65bbc387ff2..513e535381f6 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/distinct.prql --- Tokens( [ + 0..0: Start, 0..12: Comment(" mssql:test"), 12..13: NewLine, 13..17: Ident("from"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct_on.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct_on.snap index 7d3443c8e6d4..d3db05e05776 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct_on.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__distinct_on.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/distinct_on.prql --- Tokens( [ + 0..0: Start, 0..12: Comment(" mssql:test"), 12..13: NewLine, 13..17: Ident("from"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__genre_counts.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__genre_counts.snap index 196cf5589663..7ce7ffe19c4d 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__genre_counts.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__genre_counts.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/genre_counts.prql --- Tokens( [ + 0..0: Start, 0..103: Comment(" clickhouse:skip (ClickHouse prefers aliases to column names https://github.com/PRQL/prql/issues/2827)"), 103..104: NewLine, 104..116: Comment(" mssql:test"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__group_all.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__group_all.snap index 276ebaf09665..ae73cce7b3fe 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__group_all.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__group_all.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/group_all.prql --- Tokens( [ + 0..0: Start, 0..12: Comment(" mssql:test"), 12..13: NewLine, 13..17: Ident("from"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__group_sort.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__group_sort.snap index 187a9e7f43dd..3d5110e9d563 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__group_sort.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__group_sort.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/group_sort.prql --- Tokens( [ + 0..0: Start, 0..12: Comment(" mssql:test"), 12..13: NewLine, 13..17: Ident("from"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__group_sort_limit_take.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__group_sort_limit_take.snap index d6f32fe1c604..0edc25966c19 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__group_sort_limit_take.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__group_sort_limit_take.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/group_sort_limit_take.prql --- Tokens( [ + 0..0: Start, 0..62: Comment(" Compute the 3 longest songs for each genre and sort by genre"), 62..63: NewLine, 63..75: Comment(" mssql:test"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__invoice_totals.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__invoice_totals.snap index 1a1e3af65ccc..6556ea5b6588 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__invoice_totals.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__invoice_totals.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/invoice_totals.prql --- Tokens( [ + 0..0: Start, 0..56: Comment(" clickhouse:skip (clickhouse doesn't have lag function)"), 56..57: NewLine, 57..58: NewLine, diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__loop_01.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__loop_01.snap index de5d9ebf7d4a..733a6308ab36 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__loop_01.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__loop_01.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/loop_01.prql --- Tokens( [ + 0..0: Start, 0..47: Comment(" clickhouse:skip (DB::Exception: Syntax error)"), 47..48: NewLine, 48..161: Comment(" glaredb:skip (DataFusion does not support recursive CTEs https://github.com/apache/arrow-datafusion/issues/462)"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__math_module.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__math_module.snap index d49ee210ea2f..bc6aff6099e7 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__math_module.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__math_module.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/math_module.prql --- Tokens( [ + 0..0: Start, 0..12: Comment(" mssql:test"), 12..13: NewLine, 13..81: Comment(" sqlite:skip (see https://github.com/rusqlite/rusqlite/issues/1211)"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__pipelines.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__pipelines.snap index 604f9ce46ef1..06a07bfb0829 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__pipelines.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__pipelines.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/pipelines.prql --- Tokens( [ + 0..0: Start, 0..76: Comment(" sqlite:skip (Only works on Sqlite implementations which have the extension"), 76..77: NewLine, 77..88: Comment(" installed"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__read_csv.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__read_csv.snap index d0a06ad8fb9b..c7f8a18b32be 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__read_csv.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__read_csv.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/read_csv.prql --- Tokens( [ + 0..0: Start, 0..13: Comment(" sqlite:skip"), 13..14: NewLine, 14..29: Comment(" postgres:skip"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__set_ops_remove.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__set_ops_remove.snap index c68ceded492a..84e4c4b57b72 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__set_ops_remove.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__set_ops_remove.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/set_ops_remove.prql --- Tokens( [ + 0..0: Start, 0..12: Comment(" mssql:test"), 12..13: NewLine, 13..16: Keyword("let"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__sort.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__sort.snap index c119e1bac091..c0e409446480 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__sort.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__sort.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/sort.prql --- Tokens( [ + 0..0: Start, 0..12: Comment(" mssql:test"), 12..13: NewLine, 13..17: Ident("from"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__switch.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__switch.snap index 9afa06523f7d..fa6e57a571e4 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__switch.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__switch.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/switch.prql --- Tokens( [ + 0..0: Start, 0..75: Comment(" glaredb:skip (May be a bag of String type conversion for Postgres Client)"), 75..76: NewLine, 76..88: Comment(" mssql:test"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__take.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__take.snap index 1ee3caa19c9c..c0bdd9ca921d 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__take.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__take.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/take.prql --- Tokens( [ + 0..0: Start, 0..12: Comment(" mssql:test"), 12..13: NewLine, 13..17: Ident("from"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__text_module.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__text_module.snap index 47b895181050..0e769b071a53 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__text_module.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__text_module.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/text_module.prql --- Tokens( [ + 0..0: Start, 0..12: Comment(" mssql:test"), 12..13: NewLine, 13..93: Comment(" glaredb:skip — TODO: started raising an error on 2024-05-20; see `window.prql`"), diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__window.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__window.snap index ea4bb5f30d33..7b5350ad50bc 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__window.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__lex__window.snap @@ -5,6 +5,7 @@ input_file: prqlc/prqlc/tests/integration/queries/window.prql --- Tokens( [ + 0..0: Start, 0..95: Comment(" mssql:skip Conversion(\"cannot interpret I64(Some(1)) as an i32 value\")', connection.rs:200:34"), 95..96: NewLine, 96..251: Comment(" duckdb:skip problems with DISTINCT ON (duckdb internal error: [with INPUT_TYPE = int; RESULT_TYPE = unsigned char]: Assertion `min_val <= input' failed.)"), From 080a8bb6d4563ca5ae7fab1cd6b2132d722fc351 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Fri, 12 Jul 2024 00:22:36 -0700 Subject: [PATCH 14/27] tweak: Add code comment (#4722) --- prqlc/prqlc-parser/src/lexer/mod.rs | 5 ----- prqlc/prqlc-parser/src/parser/mod.rs | 5 +++-- prqlc/prqlc/src/parser.rs | 4 ++++ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/prqlc/prqlc-parser/src/lexer/mod.rs b/prqlc/prqlc-parser/src/lexer/mod.rs index f86cb319f1fa..494ea9d86e54 100644 --- a/prqlc/prqlc-parser/src/lexer/mod.rs +++ b/prqlc/prqlc-parser/src/lexer/mod.rs @@ -10,11 +10,6 @@ pub mod lr; #[cfg(test)] mod test; -// TODO: we have `lex_source` and `lex_source_recovery` and don't have the same -// structure for PR. Probably we should have a single approach to the inclusion -// and naming of a function which returns both the tokens & errors, and a -// function that returns both. - /// Lex PRQL into LR, returning both the LR and any errors encountered pub fn lex_source_recovery(source: &str, source_id: u16) -> (Option>, Vec) { let (tokens, lex_errors) = lexer().parse_recovery(source); diff --git a/prqlc/prqlc-parser/src/parser/mod.rs b/prqlc/prqlc-parser/src/parser/mod.rs index d02f5985e90c..23fa466b70f9 100644 --- a/prqlc/prqlc-parser/src/parser/mod.rs +++ b/prqlc/prqlc-parser/src/parser/mod.rs @@ -14,13 +14,14 @@ pub(crate) mod stmt; mod test; mod types; +// Note that `parse_source` is in `prqlc` crate, not in `prqlc-parser` crate, +// because it logs using the logging framework in `prqlc`. + pub fn parse_lr_to_pr( source: &str, source_id: u16, lr: Vec, ) -> (Option>, Vec) { - // We don't want comments in the AST (but we do intend to use them as part of - // formatting) let stream = prepare_stream(lr.into_iter(), source, source_id); let (pr, parse_errors) = stmt::source().parse_recovery(stream); diff --git a/prqlc/prqlc/src/parser.rs b/prqlc/prqlc/src/parser.rs index 55774272d917..692574c97e82 100644 --- a/prqlc/prqlc/src/parser.rs +++ b/prqlc/prqlc/src/parser.rs @@ -53,6 +53,10 @@ pub fn parse(file_tree: &SourceTree) -> Result { } /// Build PR AST from a PRQL query string. +// We have this function in `prqlc` rather than in `prqlc-parser` crate since +// our logging is in `prqlc` and we want to log the LR. (We could split the logging +// out into a separate crate, but it has dependencies on `prqlc` internals and +// would be an effort) pub(crate) fn parse_source(source: &str, source_id: u16) -> Result, Vec> { let (tokens, mut errors) = prqlc_parser::lexer::lex_source_recovery(source, source_id); From b72c5b2b253c328b01272d3cc6246283f9650b97 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sat, 13 Jul 2024 15:37:10 -0700 Subject: [PATCH 15/27] feat: Add DocComments to PR (#4701) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- prqlc/prqlc-parser/src/error.rs | 4 + prqlc/prqlc-parser/src/lexer/mod.rs | 3 + prqlc/prqlc-parser/src/parser/common.rs | 95 ++++++- prqlc/prqlc-parser/src/parser/expr.rs | 253 +++++++++++++----- .../prqlc-parser/src/parser/interpolation.rs | 2 + prqlc/prqlc-parser/src/parser/mod.rs | 10 +- prqlc/prqlc-parser/src/parser/perror.rs | 4 +- prqlc/prqlc-parser/src/parser/pr/expr.rs | 15 ++ prqlc/prqlc-parser/src/parser/pr/stmt.rs | 15 +- prqlc/prqlc-parser/src/parser/stmt.rs | 242 +++++++++++++++-- prqlc/prqlc-parser/src/parser/test.rs | 30 ++- ...qlc_parser__test__pipeline_parse_tree.snap | 2 +- prqlc/prqlc-parser/src/test.rs | 203 +++++++++++--- prqlc/prqlc/src/semantic/ast_expand.rs | 2 + .../tests/integration/bad_error_messages.rs | 9 +- .../prqlc/tests/integration/error_messages.rs | 2 +- ...__queries__debug_lineage__aggregation.snap | 2 +- ...n__queries__debug_lineage__arithmetic.snap | 2 +- ...gration__queries__debug_lineage__cast.snap | 2 +- ...ueries__debug_lineage__constants_only.snap | 2 +- ..._queries__debug_lineage__date_to_text.snap | 2 +- ...ion__queries__debug_lineage__distinct.snap | 2 +- ...__queries__debug_lineage__distinct_on.snap | 2 +- ..._queries__debug_lineage__genre_counts.snap | 4 +- ...on__queries__debug_lineage__group_all.snap | 2 +- ...n__queries__debug_lineage__group_sort.snap | 2 +- ..._debug_lineage__group_sort_limit_take.snap | 4 +- ...ueries__debug_lineage__invoice_totals.snap | 7 +- ...tion__queries__debug_lineage__loop_01.snap | 2 +- ...__queries__debug_lineage__math_module.snap | 2 +- ...on__queries__debug_lineage__pipelines.snap | 2 +- ...ion__queries__debug_lineage__read_csv.snap | 2 +- ...ueries__debug_lineage__set_ops_remove.snap | 4 +- ...gration__queries__debug_lineage__sort.snap | 2 +- ...ation__queries__debug_lineage__switch.snap | 2 +- ...gration__queries__debug_lineage__take.snap | 2 +- ...__queries__debug_lineage__text_module.snap | 2 +- ...ation__queries__debug_lineage__window.snap | 4 +- 38 files changed, 772 insertions(+), 176 deletions(-) diff --git a/prqlc/prqlc-parser/src/error.rs b/prqlc/prqlc-parser/src/error.rs index f9d255f856fd..9298ccc7e96d 100644 --- a/prqlc/prqlc-parser/src/error.rs +++ b/prqlc/prqlc-parser/src/error.rs @@ -46,8 +46,12 @@ pub enum MessageKind { pub enum Reason { Simple(String), Expected { + /// Where we were + // (could rename to `where` / `location` / `within`?) who: Option, + /// What we expected expected: String, + /// What we found found: String, }, Unexpected { diff --git a/prqlc/prqlc-parser/src/lexer/mod.rs b/prqlc/prqlc-parser/src/lexer/mod.rs index 494ea9d86e54..6f7a0eed1afd 100644 --- a/prqlc/prqlc-parser/src/lexer/mod.rs +++ b/prqlc/prqlc-parser/src/lexer/mod.rs @@ -184,6 +184,9 @@ fn line_wrap() -> impl Parser> { fn comment() -> impl Parser> { just('#').ignore_then(choice(( + // One option would be to check that doc comments have new lines in the + // lexer (we currently do in the parser); which would give better error + // messages? just('!').ignore_then( newline() .not() diff --git a/prqlc/prqlc-parser/src/parser/common.rs b/prqlc/prqlc-parser/src/parser/common.rs index 95b522649302..802a5c42e33e 100644 --- a/prqlc/prqlc-parser/src/parser/common.rs +++ b/prqlc/prqlc-parser/src/parser/common.rs @@ -5,8 +5,10 @@ use super::pr::{Annotation, Stmt, StmtKind}; use crate::lexer::lr::TokenKind; use crate::span::Span; -pub fn ident_part() -> impl Parser + Clone { - return select! { +use super::SupportsDocComment; + +pub(crate) fn ident_part() -> impl Parser + Clone { + select! { TokenKind::Ident(ident) => ident, TokenKind::Keyword(ident) if &ident == "module" => ident, } @@ -16,18 +18,27 @@ pub fn ident_part() -> impl Parser + Clone { [Some(TokenKind::Ident("".to_string()))], e.found().cloned(), ) - }); + }) } -pub fn keyword(kw: &'static str) -> impl Parser + Clone { +pub(crate) fn keyword(kw: &'static str) -> impl Parser + Clone { just(TokenKind::Keyword(kw.to_string())).ignored() } +/// Our approach to new lines is each item consumes new lines _before_ itself, +/// but not newlines after itself. This allows us to enforce new lines between +/// some items. The only place we handle new lines after an item is in the root +/// parser. pub fn new_line() -> impl Parser + Clone { - just(TokenKind::NewLine).ignored() + just(TokenKind::NewLine) + // Start is considered a new line, so we can enforce things start on a new + // line while allowing them to be at the beginning of a file + .or(just(TokenKind::Start)) + .ignored() + .labelled("new line") } -pub fn ctrl(char: char) -> impl Parser + Clone { +pub(crate) fn ctrl(char: char) -> impl Parser + Clone { just(TokenKind::Control(char)).ignored() } @@ -36,5 +47,77 @@ pub fn into_stmt((annotations, kind): (Vec, StmtKind), span: Span) - kind, span: Some(span), annotations, + doc_comment: None, + } +} + +pub(crate) fn doc_comment() -> impl Parser + Clone { + // doc comments must start on a new line, so we enforce a new line (which + // can also be a file start) before the doc comment + // + // TODO: we currently lose any empty newlines between doc comments; + // eventually we want to retain them + (new_line().repeated().at_least(1).ignore_then(select! { + TokenKind::DocComment(dc) => dc, + })) + .repeated() + .at_least(1) + .collect() + .map(|lines: Vec| lines.join("\n")) + .labelled("doc comment") +} + +pub(crate) fn with_doc_comment<'a, P, O>( + parser: P, +) -> impl Parser + Clone + 'a +where + P: Parser + Clone + 'a, + O: SupportsDocComment + 'a, +{ + doc_comment() + .or_not() + .then(parser) + .map(|(doc_comment, inner)| inner.with_doc_comment(doc_comment)) +} + +#[cfg(test)] +mod tests { + use insta::assert_debug_snapshot; + + use super::*; + use crate::test::parse_with_parser; + + #[test] + fn test_doc_comment() { + assert_debug_snapshot!(parse_with_parser(r#" + #! doc comment + #! another line + + "#, doc_comment()), @r###" + Ok( + " doc comment\n another line", + ) + "###); + } + + #[test] + fn test_doc_comment_or_not() { + assert_debug_snapshot!(parse_with_parser(r#"hello"#, doc_comment().or_not()).unwrap(), @"None"); + assert_debug_snapshot!(parse_with_parser(r#"hello"#, doc_comment().or_not().then_ignore(new_line().repeated()).then(ident_part())).unwrap(), @r###" + ( + None, + "hello", + ) + "###); + } + + #[test] + fn test_no_doc_comment_in_with_doc_comment() { + impl SupportsDocComment for String { + fn with_doc_comment(self, _doc_comment: Option) -> Self { + self + } + } + assert_debug_snapshot!(parse_with_parser(r#"hello"#, with_doc_comment(new_line().ignore_then(ident_part()))).unwrap(), @r###""hello""###); } } diff --git a/prqlc/prqlc-parser/src/parser/expr.rs b/prqlc/prqlc-parser/src/parser/expr.rs index 58d8bafb7b6b..278d90183046 100644 --- a/prqlc/prqlc-parser/src/parser/expr.rs +++ b/prqlc/prqlc-parser/src/parser/expr.rs @@ -3,13 +3,11 @@ use std::collections::HashMap; use chumsky::prelude::*; use itertools::Itertools; -use super::interpolation; use crate::lexer::lr::{Literal, TokenKind}; -use crate::parser::common::{ctrl, ident_part, keyword, new_line}; +use crate::parser::common::{ctrl, ident_part, keyword, new_line, with_doc_comment}; +use crate::parser::interpolation; use crate::parser::perror::PError; -use crate::parser::pr::Ident; use crate::parser::pr::*; -use crate::parser::pr::{BinOp, UnOp}; use crate::parser::types::type_expr; use crate::span::Span; @@ -30,7 +28,9 @@ pub fn expr() -> impl Parser + Clone { .map(|x| x.to_string()) .map(ExprKind::Internal); - let nested_expr = pipeline(lambda_func(expr.clone()).or(func_call(expr.clone()))).boxed(); + let nested_expr = with_doc_comment( + pipeline(lambda_func(expr.clone()).or(func_call(expr.clone()))).boxed(), + ); let tuple = tuple(nested_expr.clone()); let array = array(nested_expr.clone()); @@ -40,18 +40,20 @@ pub fn expr() -> impl Parser + Clone { let param = select! { TokenKind::Param(id) => ExprKind::Param(id) }; - let term = choice(( - literal, - internal, - tuple, - array, - interpolation, - ident_kind, - case, - param, - )) - .map_with_span(ExprKind::into_expr) - .or(pipeline_expr) + let term = with_doc_comment( + choice(( + literal, + internal, + tuple, + array, + interpolation, + ident_kind, + case, + param, + )) + .map_with_span(ExprKind::into_expr) + .or(pipeline_expr), + ) .boxed(); let term = field_lookup(term); @@ -74,12 +76,15 @@ pub fn expr() -> impl Parser + Clone { fn tuple( nested_expr: impl Parser + Clone, ) -> impl Parser + Clone { - ident_part() - .then_ignore(ctrl('=')) - .or_not() - .then(nested_expr) - .map(|(alias, expr)| Expr { alias, ..expr }) - .padded_by(new_line().repeated()) + new_line() + .repeated() + .ignore_then( + ident_part() + .then_ignore(ctrl('=')) + .or_not() + .then(nested_expr) + .map(|(alias, expr)| Expr { alias, ..expr }), + ) .separated_by(ctrl(',')) .allow_trailing() .then_ignore(new_line().repeated()) @@ -101,22 +106,25 @@ fn tuple( fn array( expr: impl Parser + Clone, ) -> impl Parser + Clone { - expr.padded_by(new_line().repeated()) - .separated_by(ctrl(',')) - .allow_trailing() - .then_ignore(new_line().repeated()) - .delimited_by(ctrl('['), ctrl(']')) - .recover_with(nested_delimiters( - TokenKind::Control('['), - TokenKind::Control(']'), - [ - (TokenKind::Control('{'), TokenKind::Control('}')), - (TokenKind::Control('('), TokenKind::Control(')')), - (TokenKind::Control('['), TokenKind::Control(']')), - ], - |_| vec![], - )) - .map(ExprKind::Array) + new_line() + .repeated() + .ignore_then( + expr.separated_by(ctrl(',')) + .allow_trailing() + .then_ignore(new_line().repeated()) + .delimited_by(ctrl('['), ctrl(']')) + .recover_with(nested_delimiters( + TokenKind::Control('['), + TokenKind::Control(']'), + [ + (TokenKind::Control('{'), TokenKind::Control('}')), + (TokenKind::Control('('), TokenKind::Control(')')), + (TokenKind::Control('['), TokenKind::Control(']')), + ], + |_| vec![], + )) + .map(ExprKind::Array), + ) .labelled("array") } @@ -158,17 +166,19 @@ fn interpolation() -> impl Parser + Clone { fn case( expr: impl Parser + Clone, ) -> impl Parser + Clone { + // The `nickname != null => nickname,` part + let mapping = func_call(expr.clone()) + .map(Box::new) + .then_ignore(just(TokenKind::ArrowFat)) + .then(func_call(expr).map(Box::new)) + .map(|(condition, value)| SwitchCase { condition, value }); + keyword("case") .ignore_then( - func_call(expr.clone()) - .map(Box::new) - .then_ignore(just(TokenKind::ArrowFat)) - .then(func_call(expr).map(Box::new)) - .map(|(condition, value)| SwitchCase { condition, value }) - .padded_by(new_line().repeated()) - .separated_by(ctrl(',')) + mapping + .separated_by(ctrl(',').then_ignore(new_line().repeated())) .allow_trailing() - .then_ignore(new_line().repeated()) + .padded_by(new_line().repeated()) .delimited_by(ctrl('['), ctrl(']')), ) .map(ExprKind::Case) @@ -274,29 +284,29 @@ where new_line().repeated().at_least(1).ignored(), )); - new_line() - .repeated() - .ignore_then( + with_doc_comment( + new_line().repeated().ignore_then( ident_part() .then_ignore(ctrl('=')) .or_not() .then(expr) - .map(|(alias, expr)| Expr { alias, ..expr }) - .separated_by(pipe) - .at_least(1) - .map_with_span(|exprs, span| { - // If there's only one expr, then we don't need to wrap it - // in a pipeline — just return the lone expr. Otherwise, - // wrap them in a pipeline. - exprs.into_iter().exactly_one().unwrap_or_else(|exprs| { - ExprKind::Pipeline(Pipeline { - exprs: exprs.collect(), - }) - .into_expr(span) - }) - }), - ) - .labelled("pipeline") + .map(|(alias, expr)| Expr { alias, ..expr }), + ), + ) + .separated_by(pipe) + .at_least(1) + .map_with_span(|exprs, span| { + // If there's only one expr, then we don't need to wrap it + // in a pipeline — just return the lone expr. Otherwise, + // wrap them in a pipeline. + exprs.into_iter().exactly_one().unwrap_or_else(|exprs| { + ExprKind::Pipeline(Pipeline { + exprs: exprs.collect(), + }) + .into_expr(span) + }) + }) + .labelled("pipeline") } fn binary_op_parser<'a, Term, Op>( @@ -543,3 +553,114 @@ fn operator_or() -> impl Parser + Clone { fn operator_coalesce() -> impl Parser + Clone { just(TokenKind::Coalesce).to(BinOp::Coalesce) } + +#[cfg(test)] +mod tests { + + use insta::assert_yaml_snapshot; + + use crate::test::{parse_with_parser, trim_start}; + + use super::*; + + #[test] + fn test_expr() { + assert_yaml_snapshot!( + parse_with_parser(r#"5+5"#, trim_start().ignore_then(expr())).unwrap(), + @r###" + --- + Binary: + left: + Literal: + Integer: 5 + span: "0:0-1" + op: Add + right: + Literal: + Integer: 5 + span: "0:2-3" + span: "0:0-3" + "###); + + assert_yaml_snapshot!( + parse_with_parser(r#"derive x = 5"#, trim_start().ignore_then(expr())).unwrap(), + @r###" + --- + Ident: derive + span: "0:0-6" + "###); + } + + #[test] + fn test_pipeline() { + assert_yaml_snapshot!( + parse_with_parser(r#" + from artists + derive x = 5 + "#, trim_start().then(pipeline(expr_call()))).unwrap(), + @r###" + --- + - ~ + - Pipeline: + exprs: + - FuncCall: + name: + Ident: from + span: "0:13-17" + args: + - Ident: artists + span: "0:18-25" + span: "0:13-25" + - FuncCall: + name: + Ident: derive + span: "0:38-44" + args: + - Literal: + Integer: 5 + span: "0:49-50" + alias: x + span: "0:38-50" + span: "0:13-50" + "###); + } + + #[test] + fn test_case() { + assert_yaml_snapshot!( + parse_with_parser(r#" + + case [ + + nickname != null => nickname, + true => null + + ] + "#, trim_start().then(case(expr()))).unwrap(), + @r###" + --- + - ~ + - Case: + - condition: + Binary: + left: + Ident: nickname + span: "0:30-38" + op: Ne + right: + Literal: "Null" + span: "0:42-46" + span: "0:30-46" + value: + Ident: nickname + span: "0:50-58" + - condition: + Literal: + Boolean: true + span: "0:72-76" + value: + Literal: "Null" + span: "0:80-84" + "###); + } +} diff --git a/prqlc/prqlc-parser/src/parser/interpolation.rs b/prqlc/prqlc-parser/src/parser/interpolation.rs index 53c91dcc1828..a5c518054f02 100644 --- a/prqlc/prqlc-parser/src/parser/interpolation.rs +++ b/prqlc/prqlc-parser/src/parser/interpolation.rs @@ -97,6 +97,7 @@ fn parse_interpolate() { 0:8-9, ), alias: None, + doc_comment: None, }, format: None, }, @@ -142,6 +143,7 @@ fn parse_interpolate() { 0:14-15, ), alias: None, + doc_comment: None, }, format: None, }, diff --git a/prqlc/prqlc-parser/src/parser/mod.rs b/prqlc/prqlc-parser/src/parser/mod.rs index 23fa466b70f9..bc102306ed47 100644 --- a/prqlc/prqlc-parser/src/parser/mod.rs +++ b/prqlc/prqlc-parser/src/parser/mod.rs @@ -1,5 +1,6 @@ use chumsky::{prelude::*, Stream}; +pub use self::common::new_line; use crate::error::Error; use crate::lexer::lr; use crate::span::Span; @@ -43,10 +44,7 @@ pub(crate) fn prepare_stream( let semantic_tokens = tokens.filter(|token| { !matches!( token.kind, - lr::TokenKind::Comment(_) - | lr::TokenKind::LineWrap(_) - | lr::TokenKind::DocComment(_) - | lr::TokenKind::Start + lr::TokenKind::Comment(_) | lr::TokenKind::LineWrap(_) ) }); @@ -61,3 +59,7 @@ pub(crate) fn prepare_stream( }; Stream::from_iter(eoi, tokens) } + +pub trait SupportsDocComment { + fn with_doc_comment(self, doc_comment: Option) -> Self; +} diff --git a/prqlc/prqlc-parser/src/parser/perror.rs b/prqlc/prqlc-parser/src/parser/perror.rs index 35056a5f091b..973aee315857 100644 --- a/prqlc/prqlc-parser/src/parser/perror.rs +++ b/prqlc/prqlc-parser/src/parser/perror.rs @@ -138,9 +138,7 @@ impl chumsky::Error for ChumError { }); self.label = self.label.merge(other.label); - for expected in other.expected { - self.expected.insert(expected); - } + self.expected.extend(other.expected); self } } diff --git a/prqlc/prqlc-parser/src/parser/pr/expr.rs b/prqlc/prqlc-parser/src/parser/pr/expr.rs index d4eb8d8c8a26..cef555076ac9 100644 --- a/prqlc/prqlc-parser/src/parser/pr/expr.rs +++ b/prqlc/prqlc-parser/src/parser/pr/expr.rs @@ -8,6 +8,7 @@ use crate::generic; use crate::lexer::lr::Literal; use crate::parser::pr::ops::{BinOp, UnOp}; use crate::parser::pr::Ty; +use crate::parser::SupportsDocComment; use crate::span::Span; impl Expr { @@ -16,6 +17,7 @@ impl Expr { kind: kind.into(), span: None, alias: None, + doc_comment: None, } } } @@ -34,6 +36,18 @@ pub struct Expr { #[serde(skip_serializing_if = "Option::is_none")] pub alias: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub doc_comment: Option, +} + +impl SupportsDocComment for Expr { + fn with_doc_comment(self, doc_comment: Option) -> Self { + Self { + doc_comment, + ..self + } + } } #[derive( @@ -81,6 +95,7 @@ impl ExprKind { span: Some(span), kind: self, alias: None, + doc_comment: None, } } } diff --git a/prqlc/prqlc-parser/src/parser/pr/stmt.rs b/prqlc/prqlc-parser/src/parser/pr/stmt.rs index faf208a4939b..878d5788a1b4 100644 --- a/prqlc/prqlc-parser/src/parser/pr/stmt.rs +++ b/prqlc/prqlc-parser/src/parser/pr/stmt.rs @@ -5,8 +5,8 @@ use schemars::JsonSchema; use semver::VersionReq; use serde::{Deserialize, Serialize}; -use crate::parser::pr::ident::Ident; use crate::parser::pr::{Expr, Ty}; +use crate::parser::{pr::ident::Ident, SupportsDocComment}; use crate::span::Span; #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default, JsonSchema)] @@ -35,6 +35,18 @@ pub struct Stmt { #[serde(skip_serializing_if = "Vec::is_empty", default)] pub annotations: Vec, + + #[serde(skip_serializing_if = "Option::is_none")] + pub doc_comment: Option, +} + +impl SupportsDocComment for Stmt { + fn with_doc_comment(self, doc_comment: Option) -> Self { + Stmt { + doc_comment, + ..self + } + } } #[derive(Debug, EnumAsInner, PartialEq, Clone, Serialize, Deserialize, JsonSchema)] @@ -85,6 +97,7 @@ impl Stmt { kind, span: None, annotations: Vec::new(), + doc_comment: None, } } } diff --git a/prqlc/prqlc-parser/src/parser/stmt.rs b/prqlc/prqlc-parser/src/parser/stmt.rs index e1fe38301e3d..23f343acf37c 100644 --- a/prqlc/prqlc-parser/src/parser/stmt.rs +++ b/prqlc/prqlc-parser/src/parser/stmt.rs @@ -4,17 +4,20 @@ use chumsky::prelude::*; use itertools::Itertools; use semver::VersionReq; -use super::common::{ctrl, ident_part, into_stmt, keyword, new_line}; +use super::common::{ctrl, ident_part, into_stmt, keyword, new_line, with_doc_comment}; use super::expr::{expr, expr_call, ident, pipeline}; use crate::lexer::lr::{Literal, TokenKind}; use crate::parser::perror::PError; use crate::parser::pr::*; use crate::parser::types::type_expr; -pub(crate) fn source() -> impl Parser, Error = PError> { - query_def() +pub fn source() -> impl Parser, Error = PError> { + with_doc_comment(query_def()) .or_not() .chain(module_contents()) + // This is the only instance we can consume newlines at the end of something, since + // this is the end of the module + .then_ignore(new_line().repeated()) .then_ignore(end()) } @@ -22,30 +25,57 @@ fn module_contents() -> impl Parser, Error = PError> { recursive(|module_contents| { let module_def = keyword("module") .ignore_then(ident_part()) - .then(module_contents.delimited_by(ctrl('{'), ctrl('}'))) + .then( + module_contents + .then_ignore(new_line().repeated()) + .delimited_by(ctrl('{'), ctrl('}')), + ) .map(|(name, stmts)| StmtKind::ModuleDef(ModuleDef { name, stmts })) .labelled("module definition"); - let annotation = just(TokenKind::Annotate) - .ignore_then(expr()) - .then_ignore(new_line().repeated()) - .map(|expr| Annotation { - expr: Box::new(expr), - }); - - annotation + let annotation = new_line() .repeated() - .then(choice((module_def, type_def(), import_def(), var_def()))) - .map_with_span(into_stmt) - .separated_by(new_line().repeated().at_least(1)) - .allow_leading() - .allow_trailing() + // TODO: we could enforce annotations starting on a new line? + // .at_least(1) + .ignore_then( + just(TokenKind::Annotate) + .ignore_then(expr()) + .map(|expr| Annotation { + expr: Box::new(expr), + }), + ) + .labelled("annotation"); + + // TODO: we want to confirm that we're not allowing things on the same + // line that should't be; e.g. `let foo = 5 let bar = 6`. We can't + // enforce a new line here because then `module two {let houses = + // both.alike}` fails (though we could force a new line after the + // `module` if that were helpful) + // + // let stmt_kind = new_line().repeated().at_least(1).ignore_then(choice(( + let stmt_kind = new_line().repeated().ignore_then(choice(( + module_def, + type_def(), + import_def(), + var_def(), + ))); + + // Currently doc comments need to be before the annotation; probably + // should relax this? + with_doc_comment( + annotation + .repeated() + .then(stmt_kind) + .map_with_span(into_stmt), + ) + .repeated() }) } fn query_def() -> impl Parser + Clone { new_line() .repeated() + .at_least(1) .ignore_then(keyword("prql")) .ignore_then( // named arg @@ -113,7 +143,9 @@ fn query_def() -> impl Parser + Clone { } fn var_def() -> impl Parser + Clone { - let let_ = keyword("let") + let let_ = new_line() + .repeated() + .ignore_then(keyword("let")) .ignore_then(ident_part()) .then(type_expr().delimited_by(ctrl('<'), ctrl('>')).or_not()) .then(ctrl('=').ignore_then(expr_call()).map(Box::new).or_not()) @@ -128,9 +160,13 @@ fn var_def() -> impl Parser + Clone { .labelled("variable definition"); let main_or_into = pipeline(expr_call()) - .then_ignore(new_line().repeated()) .map(Box::new) - .then(keyword("into").ignore_then(ident_part()).or_not()) + .then( + new_line() + .repeated() + .ignore_then(keyword("into").ignore_then(ident_part())) + .or_not(), + ) .map(|(value, name)| { let kind = if name.is_none() { VarDefKind::Main @@ -146,6 +182,8 @@ fn var_def() -> impl Parser + Clone { ty: None, }) }) + // TODO: this isn't really accurate, since a standard `from artists` + // also counts as this; we should change .labelled("variable definition"); let_.or(main_or_into) @@ -166,3 +204,167 @@ fn import_def() -> impl Parser + Clone { .map(|(alias, name)| StmtKind::ImportDef(ImportDef { name, alias })) .labelled("import statement") } + +#[cfg(test)] +mod tests { + use insta::assert_yaml_snapshot; + + use super::*; + use crate::test::parse_with_parser; + + #[test] + fn test_module_contents() { + assert_yaml_snapshot!(parse_with_parser(r#" + let world = 1 + let man = module.world + "#, module_contents()).unwrap(), @r###" + --- + - VarDef: + kind: Let + name: world + value: + Literal: + Integer: 1 + span: "0:25-26" + span: "0:0-26" + - VarDef: + kind: Let + name: man + value: + Indirection: + base: + Ident: module + span: "0:49-55" + field: + Name: world + span: "0:49-61" + span: "0:26-61" + "###); + } + + #[test] + fn test_module_def() { + // Same line + assert_yaml_snapshot!(parse_with_parser(r#"module two {let houses = both.alike} + "#, module_contents()).unwrap(), @r###" + --- + - ModuleDef: + name: two + stmts: + - VarDef: + kind: Let + name: houses + value: + Indirection: + base: + Ident: both + span: "0:25-29" + field: + Name: alike + span: "0:25-35" + span: "0:12-35" + span: "0:0-36" + "###); + + assert_yaml_snapshot!(parse_with_parser(r#" + module dignity { + let fair = 1 + let verona = we.lay + } + "#, module_contents()).unwrap(), @r###" + --- + - ModuleDef: + name: dignity + stmts: + - VarDef: + kind: Let + name: fair + value: + Literal: + Integer: 1 + span: "0:51-52" + span: "0:27-52" + - VarDef: + kind: Let + name: verona + value: + Indirection: + base: + Ident: we + span: "0:78-80" + field: + Name: lay + span: "0:78-84" + span: "0:52-84" + span: "0:0-95" + "###); + } + + #[test] + fn test_doc_comment_module() { + assert_yaml_snapshot!(parse_with_parser(r#" + + #! first doc comment + from foo + + "#, module_contents()).unwrap(), @r###" + --- + - VarDef: + kind: Main + name: main + value: + FuncCall: + name: + Ident: from + span: "0:39-43" + args: + - Ident: foo + span: "0:44-47" + span: "0:39-47" + span: "0:30-47" + doc_comment: " first doc comment" + "###); + + assert_yaml_snapshot!(parse_with_parser(r#" + + + #! first doc comment + from foo + into x + + #! second doc comment + from bar + + "#, module_contents()).unwrap(), @r###" + --- + - VarDef: + kind: Into + name: x + value: + FuncCall: + name: + Ident: from + span: "0:40-44" + args: + - Ident: foo + span: "0:45-48" + span: "0:40-48" + span: "0:31-63" + doc_comment: " first doc comment" + - VarDef: + kind: Main + name: main + value: + FuncCall: + name: + Ident: from + span: "0:103-107" + args: + - Ident: bar + span: "0:108-111" + span: "0:103-111" + span: "0:94-111" + doc_comment: " second doc comment" + "###); + } +} diff --git a/prqlc/prqlc-parser/src/parser/test.rs b/prqlc/prqlc-parser/src/parser/test.rs index 0d6ff9963f0f..2d8a71839f0f 100644 --- a/prqlc/prqlc-parser/src/parser/test.rs +++ b/prqlc/prqlc-parser/src/parser/test.rs @@ -1,3 +1,4 @@ +use chumsky::Parser; use insta::assert_yaml_snapshot; use crate::parser::prepare_stream; @@ -6,10 +7,13 @@ use crate::test::parse_with_parser; use crate::{error::Error, lexer::lex_source}; use crate::{lexer::lr::TokenKind, parser::pr::FuncCall}; -use super::pr::Expr; +use super::{common::new_line, pr::Expr}; fn parse_expr(source: &str) -> Result> { - parse_with_parser(source, super::expr::expr_call()) + parse_with_parser( + source, + new_line().repeated().ignore_then(super::expr::expr_call()), + ) } #[test] @@ -22,6 +26,8 @@ fn test_prepare_stream() { let mut stream = prepare_stream(tokens.0.into_iter(), input, 0); assert_yaml_snapshot!(stream.fetch_tokens().collect::>(), @r###" --- + - - Start + - "0:0-0" - - Ident: from - "0:0-4" - - Ident: artists @@ -1249,33 +1255,35 @@ fn test_ident_with_keywords() { #[test] fn test_case() { - assert_yaml_snapshot!(parse_expr(r#"case [ + assert_yaml_snapshot!(parse_expr(r#" + case [ nickname != null => nickname, true => null - ]"#).unwrap(), @r###" + ] + "#).unwrap(), @r###" --- Case: - condition: Binary: left: Ident: nickname - span: "0:19-27" + span: "0:28-36" op: Ne right: Literal: "Null" - span: "0:31-35" - span: "0:19-35" + span: "0:40-44" + span: "0:28-44" value: Ident: nickname - span: "0:39-47" + span: "0:48-56" - condition: Literal: Boolean: true - span: "0:61-65" + span: "0:70-74" value: Literal: "Null" - span: "0:69-73" - span: "0:0-83" + span: "0:78-82" + span: "0:9-92" "###); } diff --git a/prqlc/prqlc-parser/src/snapshots/prqlc_parser__test__pipeline_parse_tree.snap b/prqlc/prqlc-parser/src/snapshots/prqlc_parser__test__pipeline_parse_tree.snap index 84bca84aeba0..38604e50876c 100644 --- a/prqlc/prqlc-parser/src/snapshots/prqlc_parser__test__pipeline_parse_tree.snap +++ b/prqlc/prqlc-parser/src/snapshots/prqlc_parser__test__pipeline_parse_tree.snap @@ -188,4 +188,4 @@ expression: "parse_single(r#\"\nfrom employees\nfilter country == \"USA\" span: "0:711-713" span: "0:706-713" span: "0:1-713" - span: "0:1-714" + span: "0:0-713" diff --git a/prqlc/prqlc-parser/src/test.rs b/prqlc/prqlc-parser/src/test.rs index 65937607de0c..0593c577e336 100644 --- a/prqlc/prqlc-parser/src/test.rs +++ b/prqlc/prqlc-parser/src/test.rs @@ -1,6 +1,8 @@ use chumsky::Parser; use insta::{assert_debug_snapshot, assert_yaml_snapshot}; +use std::fmt::Debug; +use crate::parser::new_line; use crate::parser::pr::Stmt; use crate::parser::prepare_stream; use crate::parser::stmt; @@ -9,26 +11,34 @@ use crate::{error::Error, lexer::lr::TokenKind, parser::perror::PError}; /// Parse source code based on the supplied parser. /// /// Use this to test any parser! -pub(crate) fn parse_with_parser( +pub(crate) fn parse_with_parser( source: &str, parser: impl Parser, ) -> Result> { let tokens = crate::lexer::lex_source(source)?; let stream = prepare_stream(tokens.0.into_iter(), source, 0); - let (ast, parse_errors) = parser.parse_recovery(stream); + let (ast, parse_errors) = parser.parse_recovery_verbose(stream); if !parse_errors.is_empty() { + log::info!("ast: {ast:?}"); return Err(parse_errors.into_iter().map(|e| e.into()).collect()); } Ok(ast.unwrap()) } /// Parse into statements -fn parse_single(source: &str) -> Result, Vec> { +pub(crate) fn parse_single(source: &str) -> Result, Vec> { + // parse_with_parser(source, new_line().repeated().ignore_then(stmt::source())) parse_with_parser(source, stmt::source()) } +// TODO: move to expr singe stmts don't need it? +/// Remove leading newlines & the start token, for tests +pub(crate) fn trim_start() -> impl Parser { + new_line().repeated().ignored() +} + #[test] fn test_error_unicode_string() { // Test various unicode strings successfully parse errors. We were @@ -79,7 +89,7 @@ fn test_error_unexpected() { 0:6-7, ), reason: Simple( - "unexpected : while parsing function call", + "unexpected :", ), hints: [], code: None, @@ -363,7 +373,7 @@ fn test_basic_exprs() { - Ident: a span: "0:35-36" span: "0:28-36" - span: "0:28-36" + span: "0:0-36" "###); } @@ -541,7 +551,7 @@ fn test_function() { named_params: [] generic_type_params: [] span: "0:0-27" - span: "0:0-28" + span: "0:0-27" "###); assert_yaml_snapshot!(parse_single(r#"let count = X -> s"SUM({X})" @@ -633,7 +643,7 @@ fn test_function() { named_params: [] generic_type_params: [] span: "0:27-147" - span: "0:13-147" + span: "0:0-147" "###); assert_yaml_snapshot!(parse_single("let add = x to:a -> x + to\n").unwrap(), @r###" @@ -774,7 +784,7 @@ fn test_var_def() { SString: - String: SELECT * FROM employees span: "0:21-47" - span: "0:13-47" + span: "0:0-47" "###); assert_yaml_snapshot!(parse_single( @@ -826,7 +836,7 @@ fn test_var_def() { - Ident: x span: "0:101-102" span: "0:96-102" - span: "0:96-102" + span: "0:84-102" "###); } @@ -916,7 +926,7 @@ fn test_sql_parameters() { span: "0:37-107" span: "0:30-107" span: "0:9-107" - span: "0:9-108" + span: "0:0-107" "###); } @@ -1017,7 +1027,7 @@ join `my-proj`.`dataset`.`table` span: "0:118-126" span: "0:94-126" span: "0:1-126" - span: "0:1-127" + span: "0:0-126" "###); } @@ -1113,7 +1123,7 @@ fn test_sort() { span: "0:136-174" span: "0:131-174" span: "0:9-174" - span: "0:9-175" + span: "0:0-174" "###); } @@ -1160,7 +1170,7 @@ fn test_dates() { span: "0:39-76" span: "0:32-76" span: "0:9-76" - span: "0:9-77" + span: "0:0-76" "###); } @@ -1184,7 +1194,7 @@ fn test_multiline_string() { span: "0:20-36" alias: x span: "0:9-36" - span: "0:9-37" + span: "0:0-36" "### ) } @@ -1225,7 +1235,7 @@ derive x = 5 alias: x span: "0:14-26" span: "0:1-26" - span: "0:1-31" + span: "0:0-26" "### ) } @@ -1268,7 +1278,7 @@ fn test_coalesce() { alias: amount span: "0:32-59" span: "0:9-59" - span: "0:9-60" + span: "0:0-59" "### ) } @@ -1292,7 +1302,7 @@ fn test_literal() { span: "0:20-24" alias: x span: "0:9-24" - span: "0:9-25" + span: "0:0-24" "###) } @@ -1364,7 +1374,7 @@ fn test_allowed_idents() { span: "0:140-172" span: "0:133-172" span: "0:9-172" - span: "0:9-173" + span: "0:0-172" "###) } @@ -1457,7 +1467,7 @@ fn test_gt_lt_gte_lte() { span: "0:127-139" span: "0:120-139" span: "0:9-139" - span: "0:9-140" + span: "0:0-139" "###) } @@ -1498,7 +1508,7 @@ join s=salaries (==id) span: "0:33-37" span: "0:16-38" span: "0:1-38" - span: "0:1-39" + span: "0:0-38" "###); } @@ -1537,7 +1547,7 @@ fn test_var_defs() { value: Ident: x span: "0:17-42" - span: "0:9-42" + span: "0:0-42" "###); assert_yaml_snapshot!(parse_single(r#" @@ -1551,7 +1561,7 @@ fn test_var_defs() { value: Ident: x span: "0:9-10" - span: "0:9-25" + span: "0:0-25" "###); assert_yaml_snapshot!(parse_single(r#" @@ -1564,7 +1574,7 @@ fn test_var_defs() { value: Ident: x span: "0:9-10" - span: "0:9-11" + span: "0:0-10" "###); } @@ -1587,7 +1597,7 @@ fn test_array() { Integer: 2 span: "0:21-22" span: "0:17-24" - span: "0:9-24" + span: "0:0-24" - VarDef: kind: Let name: a @@ -1600,7 +1610,7 @@ fn test_array() { String: hello span: "0:49-56" span: "0:41-57" - span: "0:33-57" + span: "0:24-57" "###); } @@ -1635,7 +1645,7 @@ fn test_annotation() { named_params: [] generic_type_params: [] span: "0:49-61" - span: "0:9-61" + span: "0:0-61" annotations: - expr: Tuple: @@ -1647,7 +1657,8 @@ fn test_annotation() { "###); parse_single( r#" - @{binding_strength=1} let add = a b -> a + b + @{binding_strength=1} + let add = a b -> a + b "#, ) .unwrap(); @@ -1751,7 +1762,7 @@ fn test_target() { span: "0:72-77" span: "0:65-77" span: "0:45-77" - span: "0:45-78" + span: "0:34-77" "###); } @@ -1777,7 +1788,7 @@ fn test_module() { Literal: Integer: 1 span: "0:50-51" - span: "0:38-51" + span: "0:25-51" - VarDef: kind: Let name: man @@ -1789,8 +1800,8 @@ fn test_module() { field: Name: world span: "0:74-86" - span: "0:64-86" - span: "0:11-98" + span: "0:51-86" + span: "0:0-98" "###); } @@ -1804,3 +1815,133 @@ fn test_number() { ) .is_err()); } + +#[test] +fn doc_comment() { + use insta::assert_yaml_snapshot; + + assert_yaml_snapshot!(parse_single(r###" + from artists + derive x = 5 + "###).unwrap(), @r###" + --- + - VarDef: + kind: Main + name: main + value: + Pipeline: + exprs: + - FuncCall: + name: + Ident: from + span: "0:5-9" + args: + - Ident: artists + span: "0:10-17" + span: "0:5-17" + - FuncCall: + name: + Ident: derive + span: "0:22-28" + args: + - Literal: + Integer: 5 + span: "0:33-34" + alias: x + span: "0:22-34" + span: "0:5-34" + span: "0:0-34" + "###); + + assert_yaml_snapshot!(parse_single(r###" + from artists + + #! This is a doc comment + + derive x = 5 + "###).unwrap(), @r###" + --- + - VarDef: + kind: Main + name: main + value: + FuncCall: + name: + Ident: from + span: "0:5-9" + args: + - Ident: artists + span: "0:10-17" + span: "0:5-17" + span: "0:0-17" + - VarDef: + kind: Main + name: main + value: + FuncCall: + name: + Ident: derive + span: "0:53-59" + args: + - Literal: + Integer: 5 + span: "0:64-65" + alias: x + span: "0:53-65" + span: "0:47-65" + doc_comment: " This is a doc comment" + "###); + + assert_yaml_snapshot!(parse_single(r###" + #! This is a doc comment + from artists + derive x = 5 + "###).unwrap(), @r###" + --- + - VarDef: + kind: Main + name: main + value: + Pipeline: + exprs: + - FuncCall: + name: + Ident: from + span: "0:34-38" + args: + - Ident: artists + span: "0:39-46" + span: "0:34-46" + - FuncCall: + name: + Ident: derive + span: "0:51-57" + args: + - Literal: + Integer: 5 + span: "0:62-63" + alias: x + span: "0:51-63" + span: "0:34-63" + span: "0:29-63" + doc_comment: " This is a doc comment" + "###); + + assert_debug_snapshot!(parse_single(r###" + from artists #! This is a doc comment + "###).unwrap_err(), @r###" + [ + Error { + kind: Error, + span: Some( + 0:18-42, + ), + reason: Simple( + "unexpected #! This is a doc comment\n", + ), + hints: [], + code: None, + }, + ] + "###); +} diff --git a/prqlc/prqlc/src/semantic/ast_expand.rs b/prqlc/prqlc/src/semantic/ast_expand.rs index c702af614a4d..8394a41fdbba 100644 --- a/prqlc/prqlc/src/semantic/ast_expand.rs +++ b/prqlc/prqlc/src/semantic/ast_expand.rs @@ -304,6 +304,7 @@ pub fn restrict_expr(expr: pl::Expr) -> pr::Expr { kind: restrict_expr_kind(expr.kind), span: expr.span, alias: expr.alias, + doc_comment: None, } } @@ -467,6 +468,7 @@ fn restrict_stmt(stmt: pl::Stmt) -> pr::Stmt { .into_iter() .map(restrict_annotation) .collect(), + doc_comment: None, } } diff --git a/prqlc/prqlc/tests/integration/bad_error_messages.rs b/prqlc/prqlc/tests/integration/bad_error_messages.rs index 9f4b25bf2019..25e89ce05c3f 100644 --- a/prqlc/prqlc/tests/integration/bad_error_messages.rs +++ b/prqlc/prqlc/tests/integration/bad_error_messages.rs @@ -220,11 +220,12 @@ fn just_std() { std "###).unwrap_err(), @r###" Error: - ╭─[:2:5] + ╭─[:1:1] │ - 2 │ std - │ ──┬─ - │ ╰─── internal compiler error; tracked at https://github.com/PRQL/prql/issues/4474 + 1 │ ╭─▶ + 2 │ ├─▶ std + │ │ + │ ╰───────────── internal compiler error; tracked at https://github.com/PRQL/prql/issues/4474 ───╯ "###); } diff --git a/prqlc/prqlc/tests/integration/error_messages.rs b/prqlc/prqlc/tests/integration/error_messages.rs index fdc44a0983a1..ee69ca0666bf 100644 --- a/prqlc/prqlc/tests/integration/error_messages.rs +++ b/prqlc/prqlc/tests/integration/error_messages.rs @@ -98,7 +98,7 @@ fn test_errors() { │ 1 │ Answer: T-H-A-T! │ ┬ - │ ╰── unexpected : while parsing function call + │ ╰── unexpected : ───╯ "###); } diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__aggregation.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__aggregation.snap index feb3f0fbad2f..bb10ce1c8881 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__aggregation.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__aggregation.snap @@ -286,4 +286,4 @@ ast: span: 1:178-243 span: 1:168-243 span: 1:102-243 - span: 1:102-244 + span: 1:0-243 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__arithmetic.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__arithmetic.snap index 9c5a750c0f08..b4f798314b68 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__arithmetic.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__arithmetic.snap @@ -1277,4 +1277,4 @@ ast: span: 1:830-832 span: 1:825-832 span: 1:13-832 - span: 1:13-833 + span: 1:0-832 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__cast.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__cast.snap index 310465f90e8a..a5f28ab5572d 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__cast.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__cast.snap @@ -209,4 +209,4 @@ ast: span: 1:103-105 span: 1:98-105 span: 1:13-105 - span: 1:13-106 + span: 1:0-105 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__constants_only.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__constants_only.snap index df827a1fedcf..0ec1fbdecd0d 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__constants_only.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__constants_only.snap @@ -192,4 +192,4 @@ ast: alias: d span: 1:52-65 span: 1:0-65 - span: 1:0-66 + span: 1:0-65 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__date_to_text.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__date_to_text.snap index 955e7d60a22b..322da4de0e44 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__date_to_text.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__date_to_text.snap @@ -627,4 +627,4 @@ ast: span: 1:86-718 span: 1:79-718 span: 1:57-718 - span: 1:57-719 + span: 1:0-718 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct.snap index 60c1240ce602..37c8167916d4 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct.snap @@ -228,4 +228,4 @@ ast: span: 1:88-90 span: 1:77-90 span: 1:13-90 - span: 1:13-91 + span: 1:0-90 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on.snap index be7a54a80889..c3b19d9668db 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__distinct_on.snap @@ -293,4 +293,4 @@ ast: span: 1:133-159 span: 1:128-159 span: 1:13-159 - span: 1:13-160 + span: 1:0-159 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__genre_counts.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__genre_counts.snap index 3d2bdc3a0345..24159d7d24df 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__genre_counts.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__genre_counts.snap @@ -125,7 +125,7 @@ ast: span: 1:167-183 span: 1:157-183 span: 1:135-185 - span: 1:117-185 + span: 1:0-185 - VarDef: kind: Main name: main @@ -170,4 +170,4 @@ ast: alias: a span: 1:217-230 span: 1:187-230 - span: 1:187-231 + span: 1:185-230 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_all.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_all.snap index 13ab542b8168..5149c491e3af 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_all.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_all.snap @@ -339,4 +339,4 @@ ast: span: 1:152-160 span: 1:147-160 span: 1:13-160 - span: 1:13-161 + span: 1:0-160 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_sort.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_sort.snap index 7331f875ccb6..16722f0bae96 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_sort.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_sort.snap @@ -325,4 +325,4 @@ ast: span: 1:136-150 span: 1:129-150 span: 1:13-150 - span: 1:13-151 + span: 1:0-150 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_sort_limit_take.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_sort_limit_take.snap index c8e9c4bc4627..d8e94bf276d7 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_sort_limit_take.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_sort_limit_take.snap @@ -362,7 +362,7 @@ ast: Integer: 3 span: 1:168-169 span: 1:163-169 - span: 1:140-169 + span: 1:137-169 span: 1:119-171 - FuncCall: name: @@ -411,4 +411,4 @@ ast: span: 1:230-251 span: 1:225-251 span: 1:76-251 - span: 1:76-252 + span: 1:0-251 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__invoice_totals.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__invoice_totals.snap index d763b64fe62f..dadcbbb337cb 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__invoice_totals.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__invoice_totals.snap @@ -871,7 +871,7 @@ ast: alias: total_price span: 1:338-466 span: 1:328-466 - span: 1:281-466 + span: 1:276-466 span: 1:254-468 - FuncCall: name: @@ -920,7 +920,7 @@ ast: Boolean: true span: 1:521-525 span: 1:504-592 - span: 1:488-592 + span: 1:483-592 span: 1:469-594 - FuncCall: name: @@ -984,4 +984,5 @@ ast: span: 1:789-791 span: 1:784-791 span: 1:131-791 - span: 1:131-792 + span: 1:130-791 + doc_comment: ' Calculate a number of metrics about the sales of tracks in each city.' diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__loop_01.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__loop_01.snap index 8a0143ae52e8..1436d51ee9d2 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__loop_01.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__loop_01.snap @@ -363,4 +363,4 @@ ast: span: 1:255-256 span: 1:250-256 span: 1:162-256 - span: 1:162-257 + span: 1:0-256 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__math_module.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__math_module.snap index 49c84b3bf375..7b92c56f9b1d 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__math_module.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__math_module.snap @@ -945,4 +945,4 @@ ast: span: 1:110-839 span: 1:103-839 span: 1:82-839 - span: 1:82-840 + span: 1:0-839 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__pipelines.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__pipelines.snap index 705569b7ab1d..77e89cb74ca6 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__pipelines.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__pipelines.snap @@ -342,4 +342,4 @@ ast: span: 1:281-297 span: 1:274-297 span: 1:166-297 - span: 1:166-298 + span: 1:0-297 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__read_csv.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__read_csv.snap index e3b20518dfde..f12478b04ec2 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__read_csv.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__read_csv.snap @@ -74,4 +74,4 @@ ast: span: 1:97-110 span: 1:92-110 span: 1:43-110 - span: 1:43-111 + span: 1:0-110 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__set_ops_remove.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__set_ops_remove.snap index 5af44fc9bec6..0dd6dd7a607b 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__set_ops_remove.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__set_ops_remove.snap @@ -289,7 +289,7 @@ ast: named_params: [] generic_type_params: [] span: 1:28-79 - span: 1:13-79 + span: 1:0-79 - VarDef: kind: Main name: main @@ -339,4 +339,4 @@ ast: span: 1:244-245 span: 1:239-245 span: 1:81-245 - span: 1:81-246 + span: 1:79-245 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__sort.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__sort.snap index a1aa70338287..82f3ab247866 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__sort.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__sort.snap @@ -323,4 +323,4 @@ ast: span: 1:224-271 span: 1:217-271 span: 1:13-271 - span: 1:13-272 + span: 1:0-271 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__switch.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__switch.snap index c0fb2ecf4d47..957538b8f03c 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__switch.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__switch.snap @@ -234,4 +234,4 @@ ast: span: 1:252-254 span: 1:247-254 span: 1:89-254 - span: 1:89-255 + span: 1:0-254 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__take.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__take.snap index 8c02c5a85974..2b2a4492caf9 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__take.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__take.snap @@ -116,4 +116,4 @@ ast: span: 1:47-51 span: 1:42-51 span: 1:13-51 - span: 1:13-52 + span: 1:0-51 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__text_module.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__text_module.snap index 69bf68a2073c..b3cf60d07916 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__text_module.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__text_module.snap @@ -705,4 +705,4 @@ ast: span: 1:484-588 span: 1:477-588 span: 1:113-588 - span: 1:113-589 + span: 1:0-588 diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__window.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__window.snap index 999eb235b48a..8c6cc9a41d7e 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__window.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__window.snap @@ -453,7 +453,7 @@ ast: Integer: 10 span: 1:914-916 span: 1:909-916 - span: 1:793-916 + span: 1:790-916 span: 1:774-918 - FuncCall: name: @@ -502,4 +502,4 @@ ast: span: 1:1006-1020 span: 1:999-1020 span: 1:762-1020 - span: 1:762-1021 + span: 1:0-1020 From 0911ce1d7ae526c43a60e696146e146b4ca6547e Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sat, 13 Jul 2024 16:28:26 -0700 Subject: [PATCH 16/27] refactor: Generalize a `sequence` parser (#4727) --- prqlc/prqlc-parser/src/parser/common.rs | 15 ++++ prqlc/prqlc-parser/src/parser/expr.rs | 93 +++++++++++-------------- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/prqlc/prqlc-parser/src/parser/common.rs b/prqlc/prqlc-parser/src/parser/common.rs index 802a5c42e33e..24b45f24404f 100644 --- a/prqlc/prqlc-parser/src/parser/common.rs +++ b/prqlc/prqlc-parser/src/parser/common.rs @@ -80,6 +80,21 @@ where .map(|(doc_comment, inner)| inner.with_doc_comment(doc_comment)) } +/// Parse a sequence, allowing commas and new lines between items. Doesn't +/// include the surrounding delimiters. +pub(crate) fn sequence<'a, P, O>( + parser: P, +) -> impl Parser, Error = PError> + Clone + 'a +where + P: Parser + Clone + 'a, + O: 'a, +{ + parser + .separated_by(ctrl(',').then_ignore(new_line().repeated())) + .allow_trailing() + .padded_by(new_line().repeated()) +} + #[cfg(test)] mod tests { use insta::assert_debug_snapshot; diff --git a/prqlc/prqlc-parser/src/parser/expr.rs b/prqlc/prqlc-parser/src/parser/expr.rs index 278d90183046..02c7250891a6 100644 --- a/prqlc/prqlc-parser/src/parser/expr.rs +++ b/prqlc/prqlc-parser/src/parser/expr.rs @@ -11,6 +11,8 @@ use crate::parser::pr::*; use crate::parser::types::type_expr; use crate::span::Span; +use super::common::sequence; + pub fn expr_call() -> impl Parser + Clone { let expr = expr(); @@ -73,25 +75,39 @@ pub fn expr() -> impl Parser + Clone { }) } -fn tuple( - nested_expr: impl Parser + Clone, -) -> impl Parser + Clone { - new_line() - .repeated() - .ignore_then( - ident_part() - .then_ignore(ctrl('=')) - .or_not() - .then(nested_expr) - .map(|(alias, expr)| Expr { alias, ..expr }), - ) - .separated_by(ctrl(',')) - .allow_trailing() - .then_ignore(new_line().repeated()) - .delimited_by(ctrl('{'), ctrl('}')) +fn tuple<'a>( + nested_expr: impl Parser + Clone + 'a, +) -> impl Parser + Clone + 'a { + sequence( + ident_part() + .then_ignore(ctrl('=')) + .or_not() + .then(nested_expr) + .map(|(alias, expr)| Expr { alias, ..expr }), + ) + .delimited_by(ctrl('{'), ctrl('}')) + .recover_with(nested_delimiters( + TokenKind::Control('{'), + TokenKind::Control('}'), + [ + (TokenKind::Control('{'), TokenKind::Control('}')), + (TokenKind::Control('('), TokenKind::Control(')')), + (TokenKind::Control('['), TokenKind::Control(']')), + ], + |_| vec![], + )) + .map(ExprKind::Tuple) + .labelled("tuple") +} + +fn array<'a>( + expr: impl Parser + Clone + 'a, +) -> impl Parser + Clone + 'a { + sequence(expr) + .delimited_by(ctrl('['), ctrl(']')) .recover_with(nested_delimiters( - TokenKind::Control('{'), - TokenKind::Control('}'), + TokenKind::Control('['), + TokenKind::Control(']'), [ (TokenKind::Control('{'), TokenKind::Control('}')), (TokenKind::Control('('), TokenKind::Control(')')), @@ -99,32 +115,7 @@ fn tuple( ], |_| vec![], )) - .map(ExprKind::Tuple) - .labelled("tuple") -} - -fn array( - expr: impl Parser + Clone, -) -> impl Parser + Clone { - new_line() - .repeated() - .ignore_then( - expr.separated_by(ctrl(',')) - .allow_trailing() - .then_ignore(new_line().repeated()) - .delimited_by(ctrl('['), ctrl(']')) - .recover_with(nested_delimiters( - TokenKind::Control('['), - TokenKind::Control(']'), - [ - (TokenKind::Control('{'), TokenKind::Control('}')), - (TokenKind::Control('('), TokenKind::Control(')')), - (TokenKind::Control('['), TokenKind::Control(']')), - ], - |_| vec![], - )) - .map(ExprKind::Array), - ) + .map(ExprKind::Array) .labelled("array") } @@ -163,9 +154,9 @@ fn interpolation() -> impl Parser + Clone { .labelled("interpolated string") } -fn case( - expr: impl Parser + Clone, -) -> impl Parser + Clone { +fn case<'a>( + expr: impl Parser + Clone + 'a, +) -> impl Parser + Clone + 'a { // The `nickname != null => nickname,` part let mapping = func_call(expr.clone()) .map(Box::new) @@ -174,13 +165,7 @@ fn case( .map(|(condition, value)| SwitchCase { condition, value }); keyword("case") - .ignore_then( - mapping - .separated_by(ctrl(',').then_ignore(new_line().repeated())) - .allow_trailing() - .padded_by(new_line().repeated()) - .delimited_by(ctrl('['), ctrl(']')), - ) + .ignore_then(sequence(mapping).delimited_by(ctrl('['), ctrl(']'))) .map(ExprKind::Case) } From f73942093cb0fcc39b1ef2ab5722c6587b2dfd5b Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:03:21 -0700 Subject: [PATCH 17/27] refactor: Remove `common` module (#4728) --- prqlc/prqlc-parser/src/lexer/mod.rs | 2 +- prqlc/prqlc-parser/src/parser/common.rs | 138 ----------------- prqlc/prqlc-parser/src/parser/expr.rs | 10 +- .../prqlc-parser/src/parser/interpolation.rs | 5 +- prqlc/prqlc-parser/src/parser/mod.rs | 141 +++++++++++++++++- prqlc/prqlc-parser/src/parser/pr/expr.rs | 3 +- prqlc/prqlc-parser/src/parser/pr/mod.rs | 4 +- prqlc/prqlc-parser/src/parser/pr/stmt.rs | 3 +- prqlc/prqlc-parser/src/parser/stmt.rs | 2 +- prqlc/prqlc-parser/src/parser/test.rs | 5 +- prqlc/prqlc-parser/src/parser/types.rs | 4 +- prqlc/prqlc-parser/src/span.rs | 20 --- 12 files changed, 155 insertions(+), 182 deletions(-) delete mode 100644 prqlc/prqlc-parser/src/parser/common.rs diff --git a/prqlc/prqlc-parser/src/lexer/mod.rs b/prqlc/prqlc-parser/src/lexer/mod.rs index 6f7a0eed1afd..e71342bb92cc 100644 --- a/prqlc/prqlc-parser/src/lexer/mod.rs +++ b/prqlc/prqlc-parser/src/lexer/mod.rs @@ -202,7 +202,7 @@ fn comment() -> impl Parser> { ))) } -pub fn ident_part() -> impl Parser> + Clone { +pub(crate) fn ident_part() -> impl Parser> + Clone { let plain = filter(|c: &char| c.is_alphabetic() || *c == '_') .chain(filter(|c: &char| c.is_alphanumeric() || *c == '_').repeated()); diff --git a/prqlc/prqlc-parser/src/parser/common.rs b/prqlc/prqlc-parser/src/parser/common.rs deleted file mode 100644 index 24b45f24404f..000000000000 --- a/prqlc/prqlc-parser/src/parser/common.rs +++ /dev/null @@ -1,138 +0,0 @@ -use chumsky::prelude::*; - -use super::perror::PError; -use super::pr::{Annotation, Stmt, StmtKind}; -use crate::lexer::lr::TokenKind; -use crate::span::Span; - -use super::SupportsDocComment; - -pub(crate) fn ident_part() -> impl Parser + Clone { - select! { - TokenKind::Ident(ident) => ident, - TokenKind::Keyword(ident) if &ident == "module" => ident, - } - .map_err(|e: PError| { - PError::expected_input_found( - e.span(), - [Some(TokenKind::Ident("".to_string()))], - e.found().cloned(), - ) - }) -} - -pub(crate) fn keyword(kw: &'static str) -> impl Parser + Clone { - just(TokenKind::Keyword(kw.to_string())).ignored() -} - -/// Our approach to new lines is each item consumes new lines _before_ itself, -/// but not newlines after itself. This allows us to enforce new lines between -/// some items. The only place we handle new lines after an item is in the root -/// parser. -pub fn new_line() -> impl Parser + Clone { - just(TokenKind::NewLine) - // Start is considered a new line, so we can enforce things start on a new - // line while allowing them to be at the beginning of a file - .or(just(TokenKind::Start)) - .ignored() - .labelled("new line") -} - -pub(crate) fn ctrl(char: char) -> impl Parser + Clone { - just(TokenKind::Control(char)).ignored() -} - -pub fn into_stmt((annotations, kind): (Vec, StmtKind), span: Span) -> Stmt { - Stmt { - kind, - span: Some(span), - annotations, - doc_comment: None, - } -} - -pub(crate) fn doc_comment() -> impl Parser + Clone { - // doc comments must start on a new line, so we enforce a new line (which - // can also be a file start) before the doc comment - // - // TODO: we currently lose any empty newlines between doc comments; - // eventually we want to retain them - (new_line().repeated().at_least(1).ignore_then(select! { - TokenKind::DocComment(dc) => dc, - })) - .repeated() - .at_least(1) - .collect() - .map(|lines: Vec| lines.join("\n")) - .labelled("doc comment") -} - -pub(crate) fn with_doc_comment<'a, P, O>( - parser: P, -) -> impl Parser + Clone + 'a -where - P: Parser + Clone + 'a, - O: SupportsDocComment + 'a, -{ - doc_comment() - .or_not() - .then(parser) - .map(|(doc_comment, inner)| inner.with_doc_comment(doc_comment)) -} - -/// Parse a sequence, allowing commas and new lines between items. Doesn't -/// include the surrounding delimiters. -pub(crate) fn sequence<'a, P, O>( - parser: P, -) -> impl Parser, Error = PError> + Clone + 'a -where - P: Parser + Clone + 'a, - O: 'a, -{ - parser - .separated_by(ctrl(',').then_ignore(new_line().repeated())) - .allow_trailing() - .padded_by(new_line().repeated()) -} - -#[cfg(test)] -mod tests { - use insta::assert_debug_snapshot; - - use super::*; - use crate::test::parse_with_parser; - - #[test] - fn test_doc_comment() { - assert_debug_snapshot!(parse_with_parser(r#" - #! doc comment - #! another line - - "#, doc_comment()), @r###" - Ok( - " doc comment\n another line", - ) - "###); - } - - #[test] - fn test_doc_comment_or_not() { - assert_debug_snapshot!(parse_with_parser(r#"hello"#, doc_comment().or_not()).unwrap(), @"None"); - assert_debug_snapshot!(parse_with_parser(r#"hello"#, doc_comment().or_not().then_ignore(new_line().repeated()).then(ident_part())).unwrap(), @r###" - ( - None, - "hello", - ) - "###); - } - - #[test] - fn test_no_doc_comment_in_with_doc_comment() { - impl SupportsDocComment for String { - fn with_doc_comment(self, _doc_comment: Option) -> Self { - self - } - } - assert_debug_snapshot!(parse_with_parser(r#"hello"#, with_doc_comment(new_line().ignore_then(ident_part()))).unwrap(), @r###""hello""###); - } -} diff --git a/prqlc/prqlc-parser/src/parser/expr.rs b/prqlc/prqlc-parser/src/parser/expr.rs index 02c7250891a6..a4668c62c61b 100644 --- a/prqlc/prqlc-parser/src/parser/expr.rs +++ b/prqlc/prqlc-parser/src/parser/expr.rs @@ -4,22 +4,20 @@ use chumsky::prelude::*; use itertools::Itertools; use crate::lexer::lr::{Literal, TokenKind}; -use crate::parser::common::{ctrl, ident_part, keyword, new_line, with_doc_comment}; use crate::parser::interpolation; use crate::parser::perror::PError; use crate::parser::pr::*; use crate::parser::types::type_expr; +use crate::parser::{ctrl, ident_part, keyword, new_line, sequence, with_doc_comment}; use crate::span::Span; -use super::common::sequence; - -pub fn expr_call() -> impl Parser + Clone { +pub(crate) fn expr_call() -> impl Parser + Clone { let expr = expr(); lambda_func(expr.clone()).or(func_call(expr)) } -pub fn expr() -> impl Parser + Clone { +pub(crate) fn expr() -> impl Parser + Clone { recursive(|expr| { let literal = select! { TokenKind::Literal(lit) => ExprKind::Literal(lit) }; @@ -323,7 +321,7 @@ where .boxed() } -pub fn binary_op_parser_right<'a, Term, Op>( +pub(crate) fn binary_op_parser_right<'a, Term, Op>( term: Term, op: Op, ) -> impl Parser + Clone + 'a diff --git a/prqlc/prqlc-parser/src/parser/interpolation.rs b/prqlc/prqlc-parser/src/parser/interpolation.rs index a5c518054f02..962b098e4faa 100644 --- a/prqlc/prqlc-parser/src/parser/interpolation.rs +++ b/prqlc/prqlc-parser/src/parser/interpolation.rs @@ -7,7 +7,7 @@ use crate::parser::pr::*; use crate::span::{string_stream, Span}; /// Parses interpolated strings -pub fn parse(string: String, span_base: Span) -> Result, Vec> { +pub(crate) fn parse(string: String, span_base: Span) -> Result, Vec> { let prepped_stream = string_stream(string, span_base); let res = interpolated_parser().parse(prepped_stream); @@ -66,7 +66,8 @@ fn interpolated_parser() -> impl Parser, Error = Chum expr.or(string).repeated().then_ignore(end()) } -pub fn interpolate_ident_part() -> impl Parser> + Clone { +pub(crate) fn interpolate_ident_part() -> impl Parser> + Clone +{ let plain = filter(|c: &char| c.is_alphabetic() || *c == '_') .chain(filter(|c: &char| c.is_alphanumeric() || *c == '_').repeated()) .labelled("interpolated string"); diff --git a/prqlc/prqlc-parser/src/parser/mod.rs b/prqlc/prqlc-parser/src/parser/mod.rs index bc102306ed47..b572676f55e9 100644 --- a/prqlc/prqlc-parser/src/parser/mod.rs +++ b/prqlc/prqlc-parser/src/parser/mod.rs @@ -1,11 +1,12 @@ use chumsky::{prelude::*, Stream}; -pub use self::common::new_line; +use self::perror::PError; +use self::pr::{Annotation, Stmt, StmtKind}; use crate::error::Error; use crate::lexer::lr; +use crate::lexer::lr::TokenKind; use crate::span::Span; -mod common; mod expr; mod interpolation; pub(crate) mod perror; @@ -60,6 +61,140 @@ pub(crate) fn prepare_stream( Stream::from_iter(eoi, tokens) } -pub trait SupportsDocComment { +pub(crate) fn ident_part() -> impl Parser + Clone { + select! { + TokenKind::Ident(ident) => ident, + TokenKind::Keyword(ident) if &ident == "module" => ident, + } + .map_err(|e: PError| { + PError::expected_input_found( + e.span(), + [Some(TokenKind::Ident("".to_string()))], + e.found().cloned(), + ) + }) +} + +pub(crate) fn keyword(kw: &'static str) -> impl Parser + Clone { + just(TokenKind::Keyword(kw.to_string())).ignored() +} + +/// Our approach to new lines is each item consumes new lines _before_ itself, +/// but not newlines after itself. This allows us to enforce new lines between +/// some items. The only place we handle new lines after an item is in the root +/// parser. +pub(crate) fn new_line() -> impl Parser + Clone { + just(TokenKind::NewLine) + // Start is considered a new line, so we can enforce things start on a new + // line while allowing them to be at the beginning of a file + .or(just(TokenKind::Start)) + .ignored() + .labelled("new line") +} + +pub(crate) fn ctrl(char: char) -> impl Parser + Clone { + just(TokenKind::Control(char)).ignored() +} + +pub(crate) fn into_stmt((annotations, kind): (Vec, StmtKind), span: Span) -> Stmt { + Stmt { + kind, + span: Some(span), + annotations, + doc_comment: None, + } +} + +pub(crate) fn doc_comment() -> impl Parser + Clone { + // doc comments must start on a new line, so we enforce a new line (which + // can also be a file start) before the doc comment + // + // TODO: we currently lose any empty newlines between doc comments; + // eventually we want to retain them + (new_line().repeated().at_least(1).ignore_then(select! { + TokenKind::DocComment(dc) => dc, + })) + .repeated() + .at_least(1) + .collect() + .map(|lines: Vec| lines.join("\n")) + .labelled("doc comment") +} + +pub(crate) fn with_doc_comment<'a, P, O>( + parser: P, +) -> impl Parser + Clone + 'a +where + P: Parser + Clone + 'a, + O: SupportsDocComment + 'a, +{ + doc_comment() + .or_not() + .then(parser) + .map(|(doc_comment, inner)| inner.with_doc_comment(doc_comment)) +} + +/// Allows us to surround a parser by `with_doc_comment` and for a doc comment +/// to be added to the result, as long as the result implements `SupportsDocComment`. +/// +/// We could manage without it tbh, +pub(crate) trait SupportsDocComment { fn with_doc_comment(self, doc_comment: Option) -> Self; } + +/// Parse a sequence, allowing commas and new lines between items. Doesn't +/// include the surrounding delimiters. +pub(crate) fn sequence<'a, P, O>( + parser: P, +) -> impl Parser, Error = PError> + Clone + 'a +where + P: Parser + Clone + 'a, + O: 'a, +{ + parser + .separated_by(ctrl(',').then_ignore(new_line().repeated())) + .allow_trailing() + .padded_by(new_line().repeated()) +} + +#[cfg(test)] +mod tests { + use insta::assert_debug_snapshot; + + use super::*; + use crate::test::parse_with_parser; + + #[test] + fn test_doc_comment() { + assert_debug_snapshot!(parse_with_parser(r#" + #! doc comment + #! another line + + "#, doc_comment()), @r###" + Ok( + " doc comment\n another line", + ) + "###); + } + + #[test] + fn test_doc_comment_or_not() { + assert_debug_snapshot!(parse_with_parser(r#"hello"#, doc_comment().or_not()).unwrap(), @"None"); + assert_debug_snapshot!(parse_with_parser(r#"hello"#, doc_comment().or_not().then_ignore(new_line().repeated()).then(ident_part())).unwrap(), @r###" + ( + None, + "hello", + ) + "###); + } + + #[test] + fn test_no_doc_comment_in_with_doc_comment() { + impl SupportsDocComment for String { + fn with_doc_comment(self, _doc_comment: Option) -> Self { + self + } + } + assert_debug_snapshot!(parse_with_parser(r#"hello"#, with_doc_comment(new_line().ignore_then(ident_part()))).unwrap(), @r###""hello""###); + } +} diff --git a/prqlc/prqlc-parser/src/parser/pr/expr.rs b/prqlc/prqlc-parser/src/parser/pr/expr.rs index cef555076ac9..496e323fa625 100644 --- a/prqlc/prqlc-parser/src/parser/pr/expr.rs +++ b/prqlc/prqlc-parser/src/parser/pr/expr.rs @@ -4,12 +4,11 @@ use enum_as_inner::EnumAsInner; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::generic; use crate::lexer::lr::Literal; use crate::parser::pr::ops::{BinOp, UnOp}; use crate::parser::pr::Ty; -use crate::parser::SupportsDocComment; use crate::span::Span; +use crate::{generic, parser::SupportsDocComment}; impl Expr { pub fn new>(kind: K) -> Self { diff --git a/prqlc/prqlc-parser/src/parser/pr/mod.rs b/prqlc/prqlc-parser/src/parser/pr/mod.rs index 0aaa09411bf4..352388103c6b 100644 --- a/prqlc/prqlc-parser/src/parser/pr/mod.rs +++ b/prqlc/prqlc-parser/src/parser/pr/mod.rs @@ -10,9 +10,7 @@ pub use ops::*; pub use stmt::*; pub use types::*; -// re-export Literal from LR, since it's encapsulated in TyKind -//TODO: maybe remove these? pub use crate::generic; +// re-export Literal from LR, since it's encapsulated in TyKind pub use crate::lexer::lr::Literal; -pub use crate::lexer::lr::ValueAndUnit; pub use crate::span::Span; diff --git a/prqlc/prqlc-parser/src/parser/pr/stmt.rs b/prqlc/prqlc-parser/src/parser/pr/stmt.rs index 878d5788a1b4..77ff24c888bc 100644 --- a/prqlc/prqlc-parser/src/parser/pr/stmt.rs +++ b/prqlc/prqlc-parser/src/parser/pr/stmt.rs @@ -5,8 +5,9 @@ use schemars::JsonSchema; use semver::VersionReq; use serde::{Deserialize, Serialize}; +use crate::parser::pr::ident::Ident; use crate::parser::pr::{Expr, Ty}; -use crate::parser::{pr::ident::Ident, SupportsDocComment}; +use crate::parser::SupportsDocComment; use crate::span::Span; #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default, JsonSchema)] diff --git a/prqlc/prqlc-parser/src/parser/stmt.rs b/prqlc/prqlc-parser/src/parser/stmt.rs index 23f343acf37c..08675cb68d44 100644 --- a/prqlc/prqlc-parser/src/parser/stmt.rs +++ b/prqlc/prqlc-parser/src/parser/stmt.rs @@ -4,8 +4,8 @@ use chumsky::prelude::*; use itertools::Itertools; use semver::VersionReq; -use super::common::{ctrl, ident_part, into_stmt, keyword, new_line, with_doc_comment}; use super::expr::{expr, expr_call, ident, pipeline}; +use super::{ctrl, ident_part, into_stmt, keyword, new_line, with_doc_comment}; use crate::lexer::lr::{Literal, TokenKind}; use crate::parser::perror::PError; use crate::parser::pr::*; diff --git a/prqlc/prqlc-parser/src/parser/test.rs b/prqlc/prqlc-parser/src/parser/test.rs index 2d8a71839f0f..70bbf26c30bf 100644 --- a/prqlc/prqlc-parser/src/parser/test.rs +++ b/prqlc/prqlc-parser/src/parser/test.rs @@ -1,14 +1,13 @@ use chumsky::Parser; use insta::assert_yaml_snapshot; -use crate::parser::prepare_stream; +use super::prepare_stream; +use super::{new_line, pr::Expr}; use crate::span::Span; use crate::test::parse_with_parser; use crate::{error::Error, lexer::lex_source}; use crate::{lexer::lr::TokenKind, parser::pr::FuncCall}; -use super::{common::new_line, pr::Expr}; - fn parse_expr(source: &str) -> Result> { parse_with_parser( source, diff --git a/prqlc/prqlc-parser/src/parser/types.rs b/prqlc/prqlc-parser/src/parser/types.rs index b960eb0ca10a..d75f3faa655d 100644 --- a/prqlc/prqlc-parser/src/parser/types.rs +++ b/prqlc/prqlc-parser/src/parser/types.rs @@ -1,12 +1,12 @@ use chumsky::prelude::*; -use super::common::*; use super::expr::ident; use super::perror::PError; use super::pr::*; +use super::*; use crate::lexer::lr::TokenKind; -pub fn type_expr() -> impl Parser + Clone { +pub(crate) fn type_expr() -> impl Parser + Clone { recursive(|nested_type_expr| { let basic = select! { TokenKind::Literal(lit) => TyKind::Singleton(lit), diff --git a/prqlc/prqlc-parser/src/span.rs b/prqlc/prqlc-parser/src/span.rs index 6df543712706..6a8831ec9c37 100644 --- a/prqlc/prqlc-parser/src/span.rs +++ b/prqlc/prqlc-parser/src/span.rs @@ -15,26 +15,6 @@ pub struct Span { pub source_id: u16, } -impl Span { - pub fn merge_opt(a: Option, b: Option) -> Option { - match (a, b) { - (None, None) => None, - (None, Some(s)) => Some(s), - (Some(s), None) => Some(s), - (Some(a), Some(b)) => Some(Span::merge(a, b)), - } - } - - pub fn merge(a: Span, b: Span) -> Span { - assert_eq!(a.source_id, b.source_id); - Span { - start: usize::min(a.start, b.start), - end: usize::max(a.end, b.end), - source_id: a.source_id, - } - } -} - impl From for Range { fn from(a: Span) -> Self { a.start..a.end From 8a8468b0b7726532fdf374eb463fdf35f9e01a03 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Sun, 14 Jul 2024 18:27:37 +0200 Subject: [PATCH 18/27] feat: Add doc comment to documentation generator (#4729) --- CHANGELOG.md | 2 ++ prqlc/prqlc/src/cli/docs_generator.rs | 14 ++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 895205328bb6..56a3f28acd1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ - Added `prqlc debug json-schema` command to auto-generate JSON Schema representations of commonly exposed IR types such as PL and RQ. (@kgutwin, #4698) +- Add documentation comments to the output of the documentation generator. + (@vanillajonathan, #4729) **Fixes**: diff --git a/prqlc/prqlc/src/cli/docs_generator.rs b/prqlc/prqlc/src/cli/docs_generator.rs index 250ef5ff5211..4af6f5c43525 100644 --- a/prqlc/prqlc/src/cli/docs_generator.rs +++ b/prqlc/prqlc/src/cli/docs_generator.rs @@ -121,9 +121,9 @@ use prqlc::pr::{ExprKind, Stmt, StmtKind, TyKind, VarDefKind}; // var_def.name, var_def.name // )); -// //if let Some(docComment) = vardef.DocComment { -// // docs.push_str(&format!("

{docComment}

\n")); -// //} +// if let Some(doc_comment) = stmt.doc_comment { +// docs.push_str(&format!("

{doc_comment}

\n")); +// } // if let Some(expr) = &var_def.value { // match &expr.kind { @@ -261,9 +261,9 @@ Generated with [prqlc](https://prql-lang.org/) {}. docs.push_str(&format!("### {}\n", var_def.name)); - //if let Some(docComment) = vardef.DocComment { - // docs.push_str(&format!("{docComment}\n")); - //} + if let Some(doc_comment) = stmt.doc_comment { + docs.push_str(&format!("{}\n", doc_comment.trim_start())); + } docs.push('\n'); if let Some(expr) = &var_def.value { @@ -319,6 +319,7 @@ mod tests { #[test] fn generate_markdown_docs() { let input = r" + #! This is the x function. let x = arg1 arg2 -> c let fn_returns_array = -> array let fn_returns_bool = -> true @@ -354,6 +355,7 @@ mod tests { * foo ### x + This is the x function. #### Parameters * *arg1* From 0b3a52d6313302920ba8862f0ce05130b3281727 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sun, 14 Jul 2024 14:27:54 -0700 Subject: [PATCH 19/27] internal: Unify location of `generic` module (#4731) --- prqlc/prqlc-parser/src/generic.rs | 2 +- prqlc/prqlc-parser/src/parser/pr/mod.rs | 1 - prqlc/prqlc-parser/src/span.rs | 2 +- prqlc/prqlc/src/ir/generic.rs | 8 ++++---- prqlc/prqlc/src/ir/rq/expr.rs | 9 +++++---- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/prqlc/prqlc-parser/src/generic.rs b/prqlc/prqlc-parser/src/generic.rs index 029df39976a5..859ba028874e 100644 --- a/prqlc/prqlc-parser/src/generic.rs +++ b/prqlc/prqlc-parser/src/generic.rs @@ -1,4 +1,4 @@ -// Generic definitions of various AST items. +/// Generic definitions of various AST items. // // This was added in a big refactor by a generous-but-new contributor, and // hasn't been used much since, and I'm not sure carries its weight. So we diff --git a/prqlc/prqlc-parser/src/parser/pr/mod.rs b/prqlc/prqlc-parser/src/parser/pr/mod.rs index 352388103c6b..127933ccf937 100644 --- a/prqlc/prqlc-parser/src/parser/pr/mod.rs +++ b/prqlc/prqlc-parser/src/parser/pr/mod.rs @@ -10,7 +10,6 @@ pub use ops::*; pub use stmt::*; pub use types::*; -pub use crate::generic; // re-export Literal from LR, since it's encapsulated in TyKind pub use crate::lexer::lr::Literal; pub use crate::span::Span; diff --git a/prqlc/prqlc-parser/src/span.rs b/prqlc/prqlc-parser/src/span.rs index 6a8831ec9c37..03c36a746638 100644 --- a/prqlc/prqlc-parser/src/span.rs +++ b/prqlc/prqlc-parser/src/span.rs @@ -156,7 +156,7 @@ impl Sub for Span { } } -pub fn string_stream<'a>( +pub(crate) fn string_stream<'a>( s: String, span_base: Span, ) -> Stream<'a, char, Span, Box>> { diff --git a/prqlc/prqlc/src/ir/generic.rs b/prqlc/prqlc/src/ir/generic.rs index 1f8d52f0eb5b..1f8a4c8ec721 100644 --- a/prqlc/prqlc/src/ir/generic.rs +++ b/prqlc/prqlc/src/ir/generic.rs @@ -1,7 +1,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::pr; +use prqlc_parser::generic; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] pub struct ColumnSort { @@ -19,7 +19,7 @@ pub enum SortDirection { #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, JsonSchema)] pub struct WindowFrame { pub kind: WindowKind, - pub range: pr::generic::Range, + pub range: generic::Range, } #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, JsonSchema)] @@ -34,7 +34,7 @@ impl WindowFrame { self, WindowFrame { kind: WindowKind::Rows, - range: pr::generic::Range { + range: generic::Range { start: None, end: None } @@ -47,7 +47,7 @@ impl Default for WindowFrame { fn default() -> Self { Self { kind: WindowKind::Rows, - range: pr::generic::Range::unbounded(), + range: generic::Range::unbounded(), } } } diff --git a/prqlc/prqlc/src/ir/rq/expr.rs b/prqlc/prqlc/src/ir/rq/expr.rs index d9db9790505c..f08ec83e0d72 100644 --- a/prqlc/prqlc/src/ir/rq/expr.rs +++ b/prqlc/prqlc/src/ir/rq/expr.rs @@ -1,10 +1,11 @@ use enum_as_inner::EnumAsInner; +use prqlc_parser::generic; use prqlc_parser::lexer::lr::Literal; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use super::CId; -use crate::{pr, Span}; +use crate::Span; /// Analogous to [crate::ir::pl::Expr], but with fewer kinds. #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, JsonSchema)] @@ -13,9 +14,9 @@ pub struct Expr { pub span: Option, } -pub(super) type Range = pr::generic::Range; -pub(super) type InterpolateItem = pr::generic::InterpolateItem; -pub(super) type SwitchCase = pr::generic::SwitchCase; +pub(super) type Range = generic::Range; +pub(super) type InterpolateItem = generic::InterpolateItem; +pub(super) type SwitchCase = generic::SwitchCase; #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, EnumAsInner, JsonSchema)] pub enum ExprKind { From 26fa0adee946eff38183acb7a3c7fef09d90ff86 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sun, 14 Jul 2024 14:29:34 -0700 Subject: [PATCH 20/27] devops: Change PR title option to `refine` from `tweak` (#4730) --- .github/workflows/pull-request-target.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-request-target.yaml b/.github/workflows/pull-request-target.yaml index f53480a423bb..3cb15f283eec 100644 --- a/.github/workflows/pull-request-target.yaml +++ b/.github/workflows/pull-request-target.yaml @@ -25,7 +25,7 @@ jobs: # - "internal" for code quality / ergonomics improvements # - "devops" for developer ergonomics # - "web" for playground / website (but not docs) - # - "tweak" for tiny changes + # - "refine" for tiny changes types: | feat fix @@ -42,7 +42,7 @@ jobs: internal devops web - tweak + refine backport: # Backport to `web` branch on `pr-backport-web` From e984c0f83738559548a97a34e5b46b89e6d1e0e6 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sun, 14 Jul 2024 14:36:43 -0700 Subject: [PATCH 21/27] refactor: Extract `pipe` from parser (#4732) --- prqlc/prqlc-parser/src/parser/expr.rs | 9 +++------ prqlc/prqlc-parser/src/parser/mod.rs | 6 ++++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/prqlc/prqlc-parser/src/parser/expr.rs b/prqlc/prqlc-parser/src/parser/expr.rs index a4668c62c61b..d8a316700f57 100644 --- a/prqlc/prqlc-parser/src/parser/expr.rs +++ b/prqlc/prqlc-parser/src/parser/expr.rs @@ -11,6 +11,8 @@ use crate::parser::types::type_expr; use crate::parser::{ctrl, ident_part, keyword, new_line, sequence, with_doc_comment}; use crate::span::Span; +use super::pipe; + pub(crate) fn expr_call() -> impl Parser + Clone { let expr = expr(); @@ -262,11 +264,6 @@ where // expr has to be a param, because it can be either a normal expr() or a // recursive expr called from within expr(), which causes a stack overflow - let pipe = choice(( - ctrl('|').ignored(), - new_line().repeated().at_least(1).ignored(), - )); - with_doc_comment( new_line().repeated().ignore_then( ident_part() @@ -276,7 +273,7 @@ where .map(|(alias, expr)| Expr { alias, ..expr }), ), ) - .separated_by(pipe) + .separated_by(pipe()) .at_least(1) .map_with_span(|exprs, span| { // If there's only one expr, then we don't need to wrap it diff --git a/prqlc/prqlc-parser/src/parser/mod.rs b/prqlc/prqlc-parser/src/parser/mod.rs index b572676f55e9..155072dc4b03 100644 --- a/prqlc/prqlc-parser/src/parser/mod.rs +++ b/prqlc/prqlc-parser/src/parser/mod.rs @@ -157,6 +157,12 @@ where .padded_by(new_line().repeated()) } +fn pipe() -> impl Parser + Clone { + ctrl('|') + .ignored() + .or(new_line().repeated().at_least(1).ignored()) +} + #[cfg(test)] mod tests { use insta::assert_debug_snapshot; From a32d59f24e9b408c56c02b9829006a7255f25265 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sun, 14 Jul 2024 14:38:38 -0700 Subject: [PATCH 22/27] internal: Further restrict visibility of some items (#4733) --- prqlc/prqlc-parser/src/parser/mod.rs | 20 ++++++++------------ prqlc/prqlc-parser/src/parser/stmt.rs | 5 +---- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/prqlc/prqlc-parser/src/parser/mod.rs b/prqlc/prqlc-parser/src/parser/mod.rs index 155072dc4b03..9fce338af512 100644 --- a/prqlc/prqlc-parser/src/parser/mod.rs +++ b/prqlc/prqlc-parser/src/parser/mod.rs @@ -61,7 +61,7 @@ pub(crate) fn prepare_stream( Stream::from_iter(eoi, tokens) } -pub(crate) fn ident_part() -> impl Parser + Clone { +fn ident_part() -> impl Parser + Clone { select! { TokenKind::Ident(ident) => ident, TokenKind::Keyword(ident) if &ident == "module" => ident, @@ -75,7 +75,7 @@ pub(crate) fn ident_part() -> impl Parser + C }) } -pub(crate) fn keyword(kw: &'static str) -> impl Parser + Clone { +fn keyword(kw: &'static str) -> impl Parser + Clone { just(TokenKind::Keyword(kw.to_string())).ignored() } @@ -92,11 +92,11 @@ pub(crate) fn new_line() -> impl Parser + Clone { .labelled("new line") } -pub(crate) fn ctrl(char: char) -> impl Parser + Clone { +fn ctrl(char: char) -> impl Parser + Clone { just(TokenKind::Control(char)).ignored() } -pub(crate) fn into_stmt((annotations, kind): (Vec, StmtKind), span: Span) -> Stmt { +fn into_stmt((annotations, kind): (Vec, StmtKind), span: Span) -> Stmt { Stmt { kind, span: Some(span), @@ -105,7 +105,7 @@ pub(crate) fn into_stmt((annotations, kind): (Vec, StmtKind), span: } } -pub(crate) fn doc_comment() -> impl Parser + Clone { +fn doc_comment() -> impl Parser + Clone { // doc comments must start on a new line, so we enforce a new line (which // can also be a file start) before the doc comment // @@ -121,9 +121,7 @@ pub(crate) fn doc_comment() -> impl Parser + .labelled("doc comment") } -pub(crate) fn with_doc_comment<'a, P, O>( - parser: P, -) -> impl Parser + Clone + 'a +fn with_doc_comment<'a, P, O>(parser: P) -> impl Parser + Clone + 'a where P: Parser + Clone + 'a, O: SupportsDocComment + 'a, @@ -138,15 +136,13 @@ where /// to be added to the result, as long as the result implements `SupportsDocComment`. /// /// We could manage without it tbh, -pub(crate) trait SupportsDocComment { +trait SupportsDocComment { fn with_doc_comment(self, doc_comment: Option) -> Self; } /// Parse a sequence, allowing commas and new lines between items. Doesn't /// include the surrounding delimiters. -pub(crate) fn sequence<'a, P, O>( - parser: P, -) -> impl Parser, Error = PError> + Clone + 'a +fn sequence<'a, P, O>(parser: P) -> impl Parser, Error = PError> + Clone + 'a where P: Parser + Clone + 'a, O: 'a, diff --git a/prqlc/prqlc-parser/src/parser/stmt.rs b/prqlc/prqlc-parser/src/parser/stmt.rs index 08675cb68d44..6b63fdfa6400 100644 --- a/prqlc/prqlc-parser/src/parser/stmt.rs +++ b/prqlc/prqlc-parser/src/parser/stmt.rs @@ -181,10 +181,7 @@ fn var_def() -> impl Parser + Clone { value: Some(value), ty: None, }) - }) - // TODO: this isn't really accurate, since a standard `from artists` - // also counts as this; we should change - .labelled("variable definition"); + }); let_.or(main_or_into) } From f42747f5d18db52e0a9635f01ca8e79dec1cc829 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sun, 14 Jul 2024 16:43:38 -0700 Subject: [PATCH 23/27] refactor: Use `sequence` in more places (#4735) --- Cargo.lock | 2 + Cargo.toml | 2 + prqlc/prqlc-parser/Cargo.toml | 2 + prqlc/prqlc-parser/src/parser/expr.rs | 9 +-- prqlc/prqlc-parser/src/parser/mod.rs | 5 +- prqlc/prqlc-parser/src/parser/stmt.rs | 68 +++++++++++++++++-- prqlc/prqlc-parser/src/parser/test.rs | 7 +- prqlc/prqlc-parser/src/parser/types.rs | 52 +++++++------- prqlc/prqlc-parser/src/test.rs | 47 ------------- prqlc/prqlc/Cargo.toml | 4 +- ..._debug_lineage__group_sort_limit_take.snap | 2 +- ...ueries__debug_lineage__invoice_totals.snap | 4 +- ...ation__queries__debug_lineage__window.snap | 2 +- 13 files changed, 114 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f6ad43f1e5c..c05babcc0c76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2941,6 +2941,8 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "similar", + "similar-asserts", "stacker", "strum 0.26.3", ] diff --git a/Cargo.toml b/Cargo.toml index 0883bde613fa..e93ef6dd2527 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,3 +56,5 @@ semver = {version = "1.0.23", features = ["serde"]} serde = {version = "1.0.204", features = ["derive"]} serde_json = "1.0.120" serde_yaml = {version = "0.9.34"} +similar = "2.5.0" +similar-asserts = "1.5.0" diff --git a/prqlc/prqlc-parser/Cargo.toml b/prqlc/prqlc-parser/Cargo.toml index 2ebef88be120..9ba7e18bcc27 100644 --- a/prqlc/prqlc-parser/Cargo.toml +++ b/prqlc/prqlc-parser/Cargo.toml @@ -35,3 +35,5 @@ chumsky = {version = "0.9.2", features = ["ahash", "std"], default-features = fa [dev-dependencies] insta = {workspace = true} serde_json = {workspace = true} +similar = {workspace = true} +similar-asserts = {workspace = true} diff --git a/prqlc/prqlc-parser/src/parser/expr.rs b/prqlc/prqlc-parser/src/parser/expr.rs index d8a316700f57..9f88c2618688 100644 --- a/prqlc/prqlc-parser/src/parser/expr.rs +++ b/prqlc/prqlc-parser/src/parser/expr.rs @@ -122,7 +122,7 @@ fn array<'a>( fn pipeline_expr( expr: impl Parser + Clone, ) -> impl Parser + Clone { - expr.then_ignore(new_line().repeated()) + expr.padded_by(new_line().repeated()) .delimited_by(ctrl('('), ctrl(')')) .recover_with(nested_delimiters( TokenKind::Control('('), @@ -429,9 +429,9 @@ where .labelled("function call") } -fn lambda_func(expr: E) -> impl Parser + Clone +fn lambda_func<'a, E>(expr: E) -> impl Parser + Clone + 'a where - E: Parser + Clone, + E: Parser + Clone + 'a, { let param = ident_part() .then(type_expr().delimited_by(ctrl('<'), ctrl('>')).or_not()) @@ -539,7 +539,8 @@ mod tests { use insta::assert_yaml_snapshot; - use crate::test::{parse_with_parser, trim_start}; + use super::super::test::trim_start; + use crate::test::parse_with_parser; use super::*; diff --git a/prqlc/prqlc-parser/src/parser/mod.rs b/prqlc/prqlc-parser/src/parser/mod.rs index 9fce338af512..b892b154ec80 100644 --- a/prqlc/prqlc-parser/src/parser/mod.rs +++ b/prqlc/prqlc-parser/src/parser/mod.rs @@ -110,7 +110,7 @@ fn doc_comment() -> impl Parser + Clone { // can also be a file start) before the doc comment // // TODO: we currently lose any empty newlines between doc comments; - // eventually we want to retain them + // eventually we want to retain or restrict them (new_line().repeated().at_least(1).ignore_then(select! { TokenKind::DocComment(dc) => dc, })) @@ -135,7 +135,8 @@ where /// Allows us to surround a parser by `with_doc_comment` and for a doc comment /// to be added to the result, as long as the result implements `SupportsDocComment`. /// -/// We could manage without it tbh, +/// (In retrospect, we could manage without it, though probably not worth the +/// effort to remove it. We could also use it to also support Span items.) trait SupportsDocComment { fn with_doc_comment(self, doc_comment: Option) -> Self; } diff --git a/prqlc/prqlc-parser/src/parser/stmt.rs b/prqlc/prqlc-parser/src/parser/stmt.rs index 6b63fdfa6400..0bed58637e20 100644 --- a/prqlc/prqlc-parser/src/parser/stmt.rs +++ b/prqlc/prqlc-parser/src/parser/stmt.rs @@ -11,12 +11,13 @@ use crate::parser::perror::PError; use crate::parser::pr::*; use crate::parser::types::type_expr; +/// The top-level parser for a PRQL file pub fn source() -> impl Parser, Error = PError> { with_doc_comment(query_def()) .or_not() .chain(module_contents()) // This is the only instance we can consume newlines at the end of something, since - // this is the end of the module + // this is the end of the file .then_ignore(new_line().repeated()) .then_ignore(end()) } @@ -27,7 +28,7 @@ fn module_contents() -> impl Parser, Error = PError> { .ignore_then(ident_part()) .then( module_contents - .then_ignore(new_line().repeated()) + .padded_by(new_line().repeated()) .delimited_by(ctrl('{'), ctrl('}')), ) .map(|(name, stmts)| StmtKind::ModuleDef(ModuleDef { name, stmts })) @@ -50,7 +51,7 @@ fn module_contents() -> impl Parser, Error = PError> { // line that should't be; e.g. `let foo = 5 let bar = 6`. We can't // enforce a new line here because then `module two {let houses = // both.alike}` fails (though we could force a new line after the - // `module` if that were helpful) + // `module` if we wanted to?) // // let stmt_kind = new_line().repeated().at_least(1).ignore_then(choice(( let stmt_kind = new_line().repeated().ignore_then(choice(( @@ -239,6 +240,65 @@ mod tests { "###); } + #[test] + fn test_module() { + let parse_module = |s| parse_with_parser(s, module_contents()).unwrap(); + + let module_ast = parse_module( + r#" + module hello { + let world = 1 + let man = module.world + } + "#, + ); + + assert_yaml_snapshot!(module_ast, @r###" + --- + - ModuleDef: + name: hello + stmts: + - VarDef: + kind: Let + name: world + value: + Literal: + Integer: 1 + span: "0:50-51" + span: "0:38-51" + - VarDef: + kind: Let + name: man + value: + Indirection: + base: + Ident: module + span: "0:74-80" + field: + Name: world + span: "0:74-86" + span: "0:51-86" + span: "0:0-98" + "###); + + // Check this parses OK. (We tried comparing it to the AST of the result + // above, but the span information was different, so we just check it. + // Would be nice to be able to strip spans...) + parse_module( + r#" + + module hello { + + + let world = 1 + + let man = module.world + + } + "#, + ); + } + #[test] fn test_module_def() { // Same line @@ -280,7 +340,7 @@ mod tests { Literal: Integer: 1 span: "0:51-52" - span: "0:27-52" + span: "0:40-52" - VarDef: kind: Let name: verona diff --git a/prqlc/prqlc-parser/src/parser/test.rs b/prqlc/prqlc-parser/src/parser/test.rs index 70bbf26c30bf..8f39db6df197 100644 --- a/prqlc/prqlc-parser/src/parser/test.rs +++ b/prqlc/prqlc-parser/src/parser/test.rs @@ -1,8 +1,8 @@ use chumsky::Parser; use insta::assert_yaml_snapshot; -use super::prepare_stream; use super::{new_line, pr::Expr}; +use super::{perror::PError, prepare_stream}; use crate::span::Span; use crate::test::parse_with_parser; use crate::{error::Error, lexer::lex_source}; @@ -15,6 +15,11 @@ fn parse_expr(source: &str) -> Result> { ) } +/// Remove leading newlines & the start token, for tests +pub(crate) fn trim_start() -> impl Parser { + new_line().repeated().ignored() +} + #[test] fn test_prepare_stream() { use insta::assert_yaml_snapshot; diff --git a/prqlc/prqlc-parser/src/parser/types.rs b/prqlc/prqlc-parser/src/parser/types.rs index d75f3faa655d..2b57d48e3c2b 100644 --- a/prqlc/prqlc-parser/src/parser/types.rs +++ b/prqlc/prqlc-parser/src/parser/types.rs @@ -39,7 +39,7 @@ pub(crate) fn type_expr() -> impl Parser + Clone ) .map(TyKind::Function); - let tuple = choice(( + let tuple = sequence(choice(( select! { TokenKind::Range { bind_right: true, bind_left: _ } => () } .ignore_then(nested_type_expr.clone()) .map(|ty| TyTupleField::Wildcard(Some(ty))), @@ -48,10 +48,7 @@ pub(crate) fn type_expr() -> impl Parser + Clone .or_not() .then(nested_type_expr.clone()) .map(|(name, ty)| TyTupleField::Single(name, Some(ty))), - )) - .padded_by(new_line().repeated()) - .separated_by(ctrl(',')) - .allow_trailing() + ))) .delimited_by(ctrl('{'), ctrl('}')) .recover_with(nested_delimiters( TokenKind::Control('{'), @@ -81,29 +78,28 @@ pub(crate) fn type_expr() -> impl Parser + Clone let enum_ = keyword("enum") .ignore_then( - ident_part() - .then(ctrl('=').ignore_then(nested_type_expr.clone()).or_not()) - .map(|(name, ty)| { - ( - Some(name), - ty.unwrap_or_else(|| Ty::new(TyKind::Tuple(vec![]))), - ) - }) - .padded_by(new_line().repeated()) - .separated_by(ctrl(',')) - .allow_trailing() - .then_ignore(new_line().repeated()) - .delimited_by(ctrl('{'), ctrl('}')) - .recover_with(nested_delimiters( - TokenKind::Control('{'), - TokenKind::Control('}'), - [ - (TokenKind::Control('{'), TokenKind::Control('}')), - (TokenKind::Control('('), TokenKind::Control(')')), - (TokenKind::Control('['), TokenKind::Control(']')), - ], - |_| vec![], - )), + sequence( + ident_part() + .then(ctrl('=').ignore_then(nested_type_expr.clone()).or_not()) + .map(|(name, ty)| { + ( + Some(name), + ty.unwrap_or_else(|| Ty::new(TyKind::Tuple(vec![]))), + ) + }), + ) + .padded_by(new_line().repeated()) + .delimited_by(ctrl('{'), ctrl('}')) + .recover_with(nested_delimiters( + TokenKind::Control('{'), + TokenKind::Control('}'), + [ + (TokenKind::Control('{'), TokenKind::Control('}')), + (TokenKind::Control('('), TokenKind::Control(')')), + (TokenKind::Control('['), TokenKind::Control(']')), + ], + |_| vec![], + )), ) .map(TyKind::Union) .labelled("union"); diff --git a/prqlc/prqlc-parser/src/test.rs b/prqlc/prqlc-parser/src/test.rs index 0593c577e336..d3e43302882c 100644 --- a/prqlc/prqlc-parser/src/test.rs +++ b/prqlc/prqlc-parser/src/test.rs @@ -2,7 +2,6 @@ use chumsky::Parser; use insta::{assert_debug_snapshot, assert_yaml_snapshot}; use std::fmt::Debug; -use crate::parser::new_line; use crate::parser::pr::Stmt; use crate::parser::prepare_stream; use crate::parser::stmt; @@ -29,16 +28,9 @@ pub(crate) fn parse_with_parser( /// Parse into statements pub(crate) fn parse_single(source: &str) -> Result, Vec> { - // parse_with_parser(source, new_line().repeated().ignore_then(stmt::source())) parse_with_parser(source, stmt::source()) } -// TODO: move to expr singe stmts don't need it? -/// Remove leading newlines & the start token, for tests -pub(crate) fn trim_start() -> impl Parser { - new_line().repeated().ignored() -} - #[test] fn test_error_unicode_string() { // Test various unicode strings successfully parse errors. We were @@ -1766,45 +1758,6 @@ fn test_target() { "###); } -#[test] -fn test_module() { - assert_yaml_snapshot!(parse_single( - r#" - module hello { - let world = 1 - let man = module.world - } - "#, - ) - .unwrap(), @r###" - --- - - ModuleDef: - name: hello - stmts: - - VarDef: - kind: Let - name: world - value: - Literal: - Integer: 1 - span: "0:50-51" - span: "0:25-51" - - VarDef: - kind: Let - name: man - value: - Indirection: - base: - Ident: module - span: "0:74-80" - field: - Name: world - span: "0:74-86" - span: "0:51-86" - span: "0:0-98" - "###); -} - #[test] fn test_number() { // We don't allow trailing periods diff --git a/prqlc/prqlc/Cargo.toml b/prqlc/prqlc/Cargo.toml index 7a456257e6a6..b38544d9ef90 100644 --- a/prqlc/prqlc/Cargo.toml +++ b/prqlc/prqlc/Cargo.toml @@ -106,8 +106,8 @@ glob = {version = "0.3.1"} insta = {workspace = true} insta-cmd = {workspace = true} rstest = "0.21.0" -similar = {version = "2.5.0"} -similar-asserts = "1.5.0" +similar = {workspace = true} +similar-asserts = {workspace = true} tempfile = {version = "3.10.0"} test_each_file = "0.3.3" diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_sort_limit_take.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_sort_limit_take.snap index d8e94bf276d7..a994df54ac38 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_sort_limit_take.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__group_sort_limit_take.snap @@ -362,7 +362,7 @@ ast: Integer: 3 span: 1:168-169 span: 1:163-169 - span: 1:137-169 + span: 1:140-169 span: 1:119-171 - FuncCall: name: diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__invoice_totals.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__invoice_totals.snap index dadcbbb337cb..3f07fbebfc39 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__invoice_totals.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__invoice_totals.snap @@ -871,7 +871,7 @@ ast: alias: total_price span: 1:338-466 span: 1:328-466 - span: 1:276-466 + span: 1:281-466 span: 1:254-468 - FuncCall: name: @@ -920,7 +920,7 @@ ast: Boolean: true span: 1:521-525 span: 1:504-592 - span: 1:483-592 + span: 1:488-592 span: 1:469-594 - FuncCall: name: diff --git a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__window.snap b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__window.snap index 8c6cc9a41d7e..d8ae753cd517 100644 --- a/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__window.snap +++ b/prqlc/prqlc/tests/integration/snapshots/integration__queries__debug_lineage__window.snap @@ -453,7 +453,7 @@ ast: Integer: 10 span: 1:914-916 span: 1:909-916 - span: 1:790-916 + span: 1:793-916 span: 1:774-918 - FuncCall: name: From bbea4f5f9e2ef167b07c88b43c5dfdd8687afadc Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sun, 14 Jul 2024 17:38:24 -0700 Subject: [PATCH 24/27] build: Remove unused dependency (#4736) --- Cargo.lock | 2 -- prqlc/prqlc-parser/Cargo.toml | 2 -- 2 files changed, 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c05babcc0c76..7f6ad43f1e5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2941,8 +2941,6 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "similar", - "similar-asserts", "stacker", "strum 0.26.3", ] diff --git a/prqlc/prqlc-parser/Cargo.toml b/prqlc/prqlc-parser/Cargo.toml index 9ba7e18bcc27..2ebef88be120 100644 --- a/prqlc/prqlc-parser/Cargo.toml +++ b/prqlc/prqlc-parser/Cargo.toml @@ -35,5 +35,3 @@ chumsky = {version = "0.9.2", features = ["ahash", "std"], default-features = fa [dev-dependencies] insta = {workspace = true} serde_json = {workspace = true} -similar = {workspace = true} -similar-asserts = {workspace = true} From a33af911692233e06ad5c50476f32e446caa0474 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sun, 14 Jul 2024 17:53:27 -0700 Subject: [PATCH 25/27] chore: Fix nightly lint (#4737) --- web/book/src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/web/book/src/lib.rs b/web/book/src/lib.rs index d3152f3adb96..ec4dc82422ca 100644 --- a/web/book/src/lib.rs +++ b/web/book/src/lib.rs @@ -80,11 +80,11 @@ fn replace_examples(text: &str) -> Result { continue; } - lang_tags - .iter() - .filter(|tag| matches!(tag, LangTag::Other(_))) - .map(|tag| bail!("Unknown code block language: {}", tag)) - .try_collect()?; + for tag in &lang_tags { + if matches!(tag, LangTag::Other(_)) { + bail!("Unknown code block language: {}", tag) + } + } if lang_tags.contains(&LangTag::NoEval) { cmark_acc.push(event.clone()); From c58fb33a625899054bfc5ded32dadce2dc92f9fb Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Mon, 15 Jul 2024 08:50:21 -0700 Subject: [PATCH 26/27] refactor: Fix nightly lint (#4738) --- lutra/lutra/src/compile.rs | 2 +- prqlc/prqlc/src/cli/mod.rs | 2 +- prqlc/prqlc/src/lib.rs | 2 +- prqlc/prqlc/src/semantic/mod.rs | 10 +++++----- prqlc/prqlc/src/semantic/resolver/mod.rs | 5 +---- prqlc/prqlc/src/sql/operators.rs | 3 +-- prqlc/prqlc/src/sql/pq/context.rs | 3 --- prqlc/prqlc/tests/integration/resolving.rs | 2 +- 8 files changed, 11 insertions(+), 18 deletions(-) diff --git a/lutra/lutra/src/compile.rs b/lutra/lutra/src/compile.rs index ee531eaeb5c4..3348fa3086cc 100644 --- a/lutra/lutra/src/compile.rs +++ b/lutra/lutra/src/compile.rs @@ -31,7 +31,7 @@ fn parse_and_compile(source_tree: &SourceTree) -> Result Result { let ast = Some(pl.clone()); - let root_module = semantic::resolve(pl, Default::default()).map_err(ErrorMessages::from)?; + let root_module = semantic::resolve(pl).map_err(ErrorMessages::from)?; let (main, _) = root_module.find_main_rel(&[]).unwrap(); let mut fc = diff --git a/prqlc/prqlc/src/semantic/mod.rs b/prqlc/prqlc/src/semantic/mod.rs index 7349f17831e5..6e2ca359dd92 100644 --- a/prqlc/prqlc/src/semantic/mod.rs +++ b/prqlc/prqlc/src/semantic/mod.rs @@ -28,7 +28,7 @@ pub fn resolve_and_lower( main_path: &[String], database_module_path: Option<&[String]>, ) -> Result { - let root_mod = resolve(file_tree, Default::default())?; + let root_mod = resolve(file_tree)?; debug::log_stage(debug::Stage::Semantic(debug::StageSemantic::Lowering)); let default_db = [NS_DEFAULT_DB.to_string()]; @@ -40,7 +40,7 @@ pub fn resolve_and_lower( } /// Runs semantic analysis on the query. -pub fn resolve(mut module_tree: pr::ModuleDef, options: ResolverOptions) -> Result { +pub fn resolve(mut module_tree: pr::ModuleDef) -> Result { load_std_lib(&mut module_tree); // expand AST into PL @@ -53,7 +53,7 @@ pub fn resolve(mut module_tree: pr::ModuleDef, options: ResolverOptions) -> Resu module: Module::new_root(), ..Default::default() }; - let mut resolver = Resolver::new(&mut root_module, options); + let mut resolver = Resolver::new(&mut root_module); // resolve the module def into the root module debug::log_stage(debug::Stage::Semantic(debug::StageSemantic::Resolver)); @@ -86,7 +86,7 @@ pub fn load_std_lib(module_tree: &mut pr::ModuleDef) { } pub fn static_eval(expr: Expr, root_mod: &mut RootModule) -> Result { - let mut resolver = Resolver::new(root_mod, ResolverOptions::default()); + let mut resolver = Resolver::new(root_mod); resolver.static_eval_to_constant(expr) } @@ -183,7 +183,7 @@ pub mod test { pub fn parse_and_resolve(query: &str) -> Result { let source_tree = query.into(); - Ok(resolve(parse(&source_tree)?, Default::default())?) + Ok(resolve(parse(&source_tree)?)?) } #[test] diff --git a/prqlc/prqlc/src/semantic/resolver/mod.rs b/prqlc/prqlc/src/semantic/resolver/mod.rs index 36f62b8d66da..f692082b9953 100644 --- a/prqlc/prqlc/src/semantic/resolver/mod.rs +++ b/prqlc/prqlc/src/semantic/resolver/mod.rs @@ -26,8 +26,6 @@ pub struct Resolver<'a> { pub id: IdGenerator, - pub options: ResolverOptions, - pub generics: HashMap<(usize, String), Vec>, } @@ -35,10 +33,9 @@ pub struct Resolver<'a> { pub struct ResolverOptions {} impl Resolver<'_> { - pub fn new(root_mod: &mut RootModule, options: ResolverOptions) -> Resolver { + pub fn new(root_mod: &mut RootModule) -> Resolver { Resolver { root_mod, - options, current_module_path: Vec::new(), default_namespace: None, in_func_call_name: false, diff --git a/prqlc/prqlc/src/sql/operators.rs b/prqlc/prqlc/src/sql/operators.rs index b66cad1b1bc8..aba525ed2138 100644 --- a/prqlc/prqlc/src/sql/operators.rs +++ b/prqlc/prqlc/src/sql/operators.rs @@ -26,8 +26,7 @@ fn std() -> &'static decl::Module { None, ); let ast = crate::parser::parse(&std_lib).unwrap(); - let options = semantic::ResolverOptions {}; - let context = semantic::resolve(ast, options).unwrap(); + let context = semantic::resolve(ast).unwrap(); context.module }) diff --git a/prqlc/prqlc/src/sql/pq/context.rs b/prqlc/prqlc/src/sql/pq/context.rs index 8b3e85eb8c07..9089bb668553 100644 --- a/prqlc/prqlc/src/sql/pq/context.rs +++ b/prqlc/prqlc/src/sql/pq/context.rs @@ -64,8 +64,6 @@ pub enum RelationStatus { #[derive(Debug)] pub struct RelationInstance { - pub riid: RIId, - pub table_ref: TableRef, /// When a pipeline is split, [CId]s from first pipeline are assigned a new @@ -169,7 +167,6 @@ impl AnchorContext { let original_cids = table_ref.columns.iter().map(|(_, c)| *c).collect(); let relation_instance = RelationInstance { - riid, table_ref, cid_redirects, original_cids, diff --git a/prqlc/prqlc/tests/integration/resolving.rs b/prqlc/prqlc/tests/integration/resolving.rs index 35655ac1f510..170b613a96a7 100644 --- a/prqlc/prqlc/tests/integration/resolving.rs +++ b/prqlc/prqlc/tests/integration/resolving.rs @@ -7,7 +7,7 @@ fn resolve(prql_source: &str) -> Result { let sources = prqlc::SourceTree::single("".into(), prql_source.to_string()); let stmts = prqlc::prql_to_pl_tree(&sources)?; - let root_module = prqlc::semantic::resolve(stmts, Default::default()) + let root_module = prqlc::semantic::resolve(stmts) .map_err(|e| prqlc::ErrorMessages::from(e).composed(&sources))?; // resolved PL, restricted back into AST From 39d6c5ffbb09f7f364a6898c5972f5664be7f278 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Mon, 15 Jul 2024 09:42:41 -0700 Subject: [PATCH 27/27] feat: Improving error messages when parsin (#4739) --- prqlc/prqlc-parser/src/parser/expr.rs | 45 ++++++++- prqlc/prqlc-parser/src/parser/perror.rs | 9 +- prqlc/prqlc-parser/src/test.rs | 128 ++++++++++++------------ 3 files changed, 115 insertions(+), 67 deletions(-) diff --git a/prqlc/prqlc-parser/src/parser/expr.rs b/prqlc/prqlc-parser/src/parser/expr.rs index 9f88c2618688..bb3450f3e23d 100644 --- a/prqlc/prqlc-parser/src/parser/expr.rs +++ b/prqlc/prqlc-parser/src/parser/expr.rs @@ -537,13 +537,56 @@ fn operator_coalesce() -> impl Parser + Clone #[cfg(test)] mod tests { - use insta::assert_yaml_snapshot; + use insta::{assert_debug_snapshot, assert_yaml_snapshot}; use super::super::test::trim_start; use crate::test::parse_with_parser; use super::*; + #[test] + fn test_tuple() { + let tuple = || trim_start().ignore_then(tuple(expr())); + assert_yaml_snapshot!( + parse_with_parser(r#"{a = 5, b = 6}"#, tuple()).unwrap(), + @r###" + --- + Tuple: + - Literal: + Integer: 5 + span: "0:5-6" + alias: a + - Literal: + Integer: 6 + span: "0:12-13" + alias: b + "###); + + assert_debug_snapshot!( + parse_with_parser(r#" + {a = 5 + b = 6}"#, tuple()).unwrap_err(), + @r###" + [ + Error { + kind: Error, + span: Some( + 0:33-34, + ), + reason: Expected { + who: Some( + "new line", + ), + expected: "}", + found: "b", + }, + hints: [], + code: None, + }, + ] + "###); + } + #[test] fn test_expr() { assert_yaml_snapshot!( diff --git a/prqlc/prqlc-parser/src/parser/perror.rs b/prqlc/prqlc-parser/src/parser/perror.rs index 973aee315857..aebe54457cfc 100644 --- a/prqlc/prqlc-parser/src/parser/perror.rs +++ b/prqlc/prqlc-parser/src/parser/perror.rs @@ -222,8 +222,13 @@ impl From for Error { .all(|t| matches!(t, None | Some(TokenKind::NewLine))); let expected: Vec = e .expected() - // Only include whitespace if we're _only_ expecting whitespace - .filter(|t| is_all_whitespace || !matches!(t, None | Some(TokenKind::NewLine))) + // Only include whitespace if we're _only_ expecting whitespace, + // otherwise we get "expected start of line or new line or }" + // when there's no ending `}`, since a new line is allowed. + .filter(|t| { + is_all_whitespace + || !matches!(t, None | Some(TokenKind::NewLine) | Some(TokenKind::Start)) + }) .cloned() .map(token_to_string) .collect(); diff --git a/prqlc/prqlc-parser/src/test.rs b/prqlc/prqlc-parser/src/test.rs index d3e43302882c..40100281b742 100644 --- a/prqlc/prqlc-parser/src/test.rs +++ b/prqlc/prqlc-parser/src/test.rs @@ -27,7 +27,7 @@ pub(crate) fn parse_with_parser( } /// Parse into statements -pub(crate) fn parse_single(source: &str) -> Result, Vec> { +pub(crate) fn parse_source(source: &str) -> Result, Vec> { parse_with_parser(source, stmt::source()) } @@ -35,15 +35,15 @@ pub(crate) fn parse_single(source: &str) -> Result, Vec> { fn test_error_unicode_string() { // Test various unicode strings successfully parse errors. We were // getting loops in the lexer before. - parse_single("s’ ").unwrap_err(); - parse_single("s’").unwrap_err(); - parse_single(" s’").unwrap_err(); - parse_single(" ’ s").unwrap_err(); - parse_single("’s").unwrap_err(); - parse_single("👍 s’").unwrap_err(); + parse_source("s’ ").unwrap_err(); + parse_source("s’").unwrap_err(); + parse_source(" s’").unwrap_err(); + parse_source(" ’ s").unwrap_err(); + parse_source("’s").unwrap_err(); + parse_source("👍 s’").unwrap_err(); let source = "Mississippi has four S’s and four I’s."; - assert_debug_snapshot!(parse_single(source).unwrap_err(), @r###" + assert_debug_snapshot!(parse_source(source).unwrap_err(), @r###" [ Error { kind: Error, @@ -73,7 +73,7 @@ fn test_error_unicode_string() { #[test] fn test_error_unexpected() { - assert_debug_snapshot!(parse_single("Answer: T-H-A-T!").unwrap_err(), @r###" + assert_debug_snapshot!(parse_source("Answer: T-H-A-T!").unwrap_err(), @r###" [ Error { kind: Error, @@ -92,7 +92,7 @@ fn test_error_unexpected() { #[test] fn test_pipeline_parse_tree() { - assert_yaml_snapshot!(parse_single( + assert_yaml_snapshot!(parse_source( r#" from employees filter country == "USA" # Each line transforms the previous result. @@ -122,9 +122,9 @@ take 20 #[test] fn test_take() { - parse_single("take 10").unwrap(); + parse_source("take 10").unwrap(); - assert_yaml_snapshot!(parse_single(r#"take 10"#).unwrap(), @r###" + assert_yaml_snapshot!(parse_source(r#"take 10"#).unwrap(), @r###" --- - VarDef: kind: Main @@ -142,7 +142,7 @@ fn test_take() { span: "0:0-7" "###); - assert_yaml_snapshot!(parse_single(r#"take ..10"#).unwrap(), @r###" + assert_yaml_snapshot!(parse_source(r#"take ..10"#).unwrap(), @r###" --- - VarDef: kind: Main @@ -164,7 +164,7 @@ fn test_take() { span: "0:0-9" "###); - assert_yaml_snapshot!(parse_single(r#"take 1..10"#).unwrap(), @r###" + assert_yaml_snapshot!(parse_source(r#"take 1..10"#).unwrap(), @r###" --- - VarDef: kind: Main @@ -193,7 +193,7 @@ fn test_take() { #[test] fn test_filter() { assert_yaml_snapshot!( - parse_single(r#"filter country == "USA""#).unwrap(), @r###" + parse_source(r#"filter country == "USA""#).unwrap(), @r###" --- - VarDef: kind: Main @@ -219,7 +219,7 @@ fn test_filter() { "###); assert_yaml_snapshot!( - parse_single(r#"filter (text.upper country) == "USA""#).unwrap(), @r###" + parse_source(r#"filter (text.upper country) == "USA""#).unwrap(), @r###" --- - VarDef: kind: Main @@ -259,7 +259,7 @@ fn test_filter() { #[test] fn test_aggregate() { - let aggregate = parse_single( + let aggregate = parse_source( r"group {title} ( aggregate {sum salary, count} )", @@ -302,7 +302,7 @@ fn test_aggregate() { span: "0:0-77" span: "0:0-77" "###); - let aggregate = parse_single( + let aggregate = parse_source( r"group {title} ( aggregate {sum salary} )", @@ -348,7 +348,7 @@ fn test_aggregate() { #[test] fn test_basic_exprs() { // Currently not putting comments in our parse tree, so this is blank. - assert_yaml_snapshot!(parse_single( + assert_yaml_snapshot!(parse_source( r#"# this is a comment select a"# ).unwrap(), @r###" @@ -371,7 +371,7 @@ fn test_basic_exprs() { #[test] fn test_function() { - assert_yaml_snapshot!(parse_single("let plus_one = x -> x + 1\n").unwrap(), @r###" + assert_yaml_snapshot!(parse_source("let plus_one = x -> x + 1\n").unwrap(), @r###" --- - VarDef: kind: Let @@ -398,7 +398,7 @@ fn test_function() { span: "0:15-26" span: "0:0-26" "###); - assert_yaml_snapshot!(parse_single("let identity = x -> x\n").unwrap() + assert_yaml_snapshot!(parse_source("let identity = x -> x\n").unwrap() , @r###" --- - VarDef: @@ -418,7 +418,7 @@ fn test_function() { span: "0:15-22" span: "0:0-22" "###); - assert_yaml_snapshot!(parse_single("let plus_one = x -> (x + 1)\n").unwrap() + assert_yaml_snapshot!(parse_source("let plus_one = x -> (x + 1)\n").unwrap() , @r###" --- - VarDef: @@ -446,7 +446,7 @@ fn test_function() { span: "0:15-28" span: "0:0-28" "###); - assert_yaml_snapshot!(parse_single("let plus_one = x -> x + 1\n").unwrap() + assert_yaml_snapshot!(parse_source("let plus_one = x -> x + 1\n").unwrap() , @r###" --- - VarDef: @@ -475,7 +475,7 @@ fn test_function() { span: "0:0-26" "###); - assert_yaml_snapshot!(parse_single("let foo = x -> some_func (foo bar + 1) (plax) - baz\n").unwrap() + assert_yaml_snapshot!(parse_source("let foo = x -> some_func (foo bar + 1) (plax) - baz\n").unwrap() , @r###" --- - VarDef: @@ -525,7 +525,7 @@ fn test_function() { span: "0:0-51" "###); - assert_yaml_snapshot!(parse_single("func return_constant -> 42\n").unwrap(), @r###" + assert_yaml_snapshot!(parse_source("func return_constant -> 42\n").unwrap(), @r###" --- - VarDef: kind: Main @@ -546,7 +546,7 @@ fn test_function() { span: "0:0-27" "###); - assert_yaml_snapshot!(parse_single(r#"let count = X -> s"SUM({X})" + assert_yaml_snapshot!(parse_source(r#"let count = X -> s"SUM({X})" "#).unwrap(), @r###" --- - VarDef: @@ -574,7 +574,7 @@ fn test_function() { span: "0:0-28" "###); - assert_yaml_snapshot!(parse_single( + assert_yaml_snapshot!(parse_source( r#" let lag_day = x -> ( window x @@ -638,7 +638,7 @@ fn test_function() { span: "0:0-147" "###); - assert_yaml_snapshot!(parse_single("let add = x to:a -> x + to\n").unwrap(), @r###" + assert_yaml_snapshot!(parse_source("let add = x to:a -> x + to\n").unwrap(), @r###" --- - VarDef: kind: Let @@ -672,7 +672,7 @@ fn test_function() { #[test] fn test_var_def() { - assert_yaml_snapshot!(parse_single( + assert_yaml_snapshot!(parse_source( "let newest_employees = (from employees)" ).unwrap(), @r###" --- @@ -691,7 +691,7 @@ fn test_var_def() { span: "0:0-39" "###); - assert_yaml_snapshot!(parse_single( + assert_yaml_snapshot!(parse_source( r#" let newest_employees = ( from employees @@ -765,7 +765,7 @@ fn test_var_def() { span: "0:0-231" "###); - assert_yaml_snapshot!(parse_single(r#" + assert_yaml_snapshot!(parse_source(r#" let e = s"SELECT * FROM employees" "#).unwrap(), @r###" --- @@ -779,7 +779,7 @@ fn test_var_def() { span: "0:0-47" "###); - assert_yaml_snapshot!(parse_single( + assert_yaml_snapshot!(parse_source( "let x = ( from x_table @@ -834,7 +834,7 @@ fn test_var_def() { #[test] fn test_inline_pipeline() { - assert_yaml_snapshot!(parse_single("let median = x -> (x | percentile 50)\n").unwrap(), @r###" + assert_yaml_snapshot!(parse_source("let median = x -> (x | percentile 50)\n").unwrap(), @r###" --- - VarDef: kind: Let @@ -869,7 +869,7 @@ fn test_inline_pipeline() { #[test] fn test_sql_parameters() { - assert_yaml_snapshot!(parse_single(r#" + assert_yaml_snapshot!(parse_source(r#" from mytable filter { first_name == $1, @@ -925,7 +925,7 @@ fn test_sql_parameters() { #[test] fn test_tab_characters() { // #284 - parse_single( + parse_source( "from c_invoice join doc:c_doctype (==c_invoice_id) select [ @@ -946,7 +946,7 @@ join `my-proj.dataset.table` join `my-proj`.`dataset`.`table` "; - assert_yaml_snapshot!(parse_single(prql).unwrap(), @r###" + assert_yaml_snapshot!(parse_source(prql).unwrap(), @r###" --- - VarDef: kind: Main @@ -1025,7 +1025,7 @@ join `my-proj`.`dataset`.`table` #[test] fn test_sort() { - assert_yaml_snapshot!(parse_single(" + assert_yaml_snapshot!(parse_source(" from invoices sort issued_at sort (-issued_at) @@ -1121,7 +1121,7 @@ fn test_sort() { #[test] fn test_dates() { - assert_yaml_snapshot!(parse_single(" + assert_yaml_snapshot!(parse_source(" from employees derive {age_plus_two_years = (age + 2years)} ").unwrap(), @r###" @@ -1168,7 +1168,7 @@ fn test_dates() { #[test] fn test_multiline_string() { - assert_yaml_snapshot!(parse_single(r##" + assert_yaml_snapshot!(parse_source(r##" derive x = r"r-string test" "##).unwrap(), @r###" --- @@ -1194,7 +1194,7 @@ fn test_multiline_string() { fn test_empty_lines() { // The span of the Pipeline shouldn't include the empty lines; the VarDef // should have a larger span - assert_yaml_snapshot!(parse_single(r#" + assert_yaml_snapshot!(parse_source(r#" from artists derive x = 5 @@ -1233,7 +1233,7 @@ derive x = 5 #[test] fn test_coalesce() { - assert_yaml_snapshot!(parse_single(r###" + assert_yaml_snapshot!(parse_source(r###" from employees derive amount = amount ?? 0 "###).unwrap(), @r###" @@ -1276,7 +1276,7 @@ fn test_coalesce() { #[test] fn test_literal() { - assert_yaml_snapshot!(parse_single(r###" + assert_yaml_snapshot!(parse_source(r###" derive x = true "###).unwrap(), @r###" --- @@ -1300,7 +1300,7 @@ fn test_literal() { #[test] fn test_allowed_idents() { - assert_yaml_snapshot!(parse_single(r###" + assert_yaml_snapshot!(parse_source(r###" from employees join _salary (==employee_id) # table with leading underscore filter first_name == $1 @@ -1372,7 +1372,7 @@ fn test_allowed_idents() { #[test] fn test_gt_lt_gte_lte() { - assert_yaml_snapshot!(parse_single(r###" + assert_yaml_snapshot!(parse_source(r###" from people filter age >= 100 filter num_grandchildren <= 10 @@ -1465,7 +1465,7 @@ fn test_gt_lt_gte_lte() { #[test] fn test_assign() { - assert_yaml_snapshot!(parse_single(r###" + assert_yaml_snapshot!(parse_source(r###" from employees join s=salaries (==id) "###).unwrap(), @r###" @@ -1507,7 +1507,7 @@ join s=salaries (==id) #[test] fn test_unicode() { let source = "from tète"; - assert_yaml_snapshot!(parse_single(source).unwrap(), @r###" + assert_yaml_snapshot!(parse_source(source).unwrap(), @r###" --- - VarDef: kind: Main @@ -1527,7 +1527,7 @@ fn test_unicode() { #[test] fn test_var_defs() { - assert_yaml_snapshot!(parse_single(r#" + assert_yaml_snapshot!(parse_source(r#" let a = ( x ) @@ -1542,7 +1542,7 @@ fn test_var_defs() { span: "0:0-42" "###); - assert_yaml_snapshot!(parse_single(r#" + assert_yaml_snapshot!(parse_source(r#" x into a "#).unwrap(), @r###" @@ -1556,7 +1556,7 @@ fn test_var_defs() { span: "0:0-25" "###); - assert_yaml_snapshot!(parse_single(r#" + assert_yaml_snapshot!(parse_source(r#" x "#).unwrap(), @r###" --- @@ -1572,7 +1572,7 @@ fn test_var_defs() { #[test] fn test_array() { - assert_yaml_snapshot!(parse_single(r#" + assert_yaml_snapshot!(parse_source(r#" let a = [1, 2,] let a = [false, "hello"] "#).unwrap(), @r###" @@ -1608,7 +1608,7 @@ fn test_array() { #[test] fn test_annotation() { - assert_yaml_snapshot!(parse_single(r#" + assert_yaml_snapshot!(parse_source(r#" @{binding_strength=1} let add = a b -> a + b "#).unwrap(), @r###" @@ -1647,7 +1647,7 @@ fn test_annotation() { alias: binding_strength span: "0:10-30" "###); - parse_single( + parse_source( r#" @{binding_strength=1} let add = a b -> a + b @@ -1655,7 +1655,7 @@ fn test_annotation() { ) .unwrap(); - parse_single( + parse_source( r#" @{binding_strength=1} # comment @@ -1664,7 +1664,7 @@ fn test_annotation() { ) .unwrap(); - parse_single( + parse_source( r#" @{binding_strength=1} @@ -1683,7 +1683,7 @@ fn check_valid_version() { "#, env!("CARGO_PKG_VERSION_MAJOR") ); - assert!(parse_single(&stmt).is_ok()); + assert!(parse_source(&stmt).is_ok()); let stmt = format!( r#" @@ -1692,7 +1692,7 @@ fn check_valid_version() { env!("CARGO_PKG_VERSION_MAJOR"), env!("CARGO_PKG_VERSION_MINOR") ); - assert!(parse_single(&stmt).is_ok()); + assert!(parse_source(&stmt).is_ok()); let stmt = format!( r#" @@ -1702,7 +1702,7 @@ fn check_valid_version() { env!("CARGO_PKG_VERSION_MINOR"), env!("CARGO_PKG_VERSION_PATCH"), ); - assert!(parse_single(&stmt).is_ok()); + assert!(parse_source(&stmt).is_ok()); } #[test] @@ -1711,12 +1711,12 @@ fn check_invalid_version() { "prql version:{}\n", env!("CARGO_PKG_VERSION_MAJOR").parse::().unwrap() + 1 ); - assert!(parse_single(&stmt).is_err()); + assert!(parse_source(&stmt).is_err()); } #[test] fn test_target() { - assert_yaml_snapshot!(parse_single( + assert_yaml_snapshot!(parse_source( r#" prql target:sql.sqlite @@ -1761,7 +1761,7 @@ fn test_target() { #[test] fn test_number() { // We don't allow trailing periods - assert!(parse_single( + assert!(parse_source( r#" from artists derive x = 1."# @@ -1773,7 +1773,7 @@ fn test_number() { fn doc_comment() { use insta::assert_yaml_snapshot; - assert_yaml_snapshot!(parse_single(r###" + assert_yaml_snapshot!(parse_source(r###" from artists derive x = 5 "###).unwrap(), @r###" @@ -1806,7 +1806,7 @@ fn doc_comment() { span: "0:0-34" "###); - assert_yaml_snapshot!(parse_single(r###" + assert_yaml_snapshot!(parse_source(r###" from artists #! This is a doc comment @@ -1845,7 +1845,7 @@ fn doc_comment() { doc_comment: " This is a doc comment" "###); - assert_yaml_snapshot!(parse_single(r###" + assert_yaml_snapshot!(parse_source(r###" #! This is a doc comment from artists derive x = 5 @@ -1880,7 +1880,7 @@ fn doc_comment() { doc_comment: " This is a doc comment" "###); - assert_debug_snapshot!(parse_single(r###" + assert_debug_snapshot!(parse_source(r###" from artists #! This is a doc comment "###).unwrap_err(), @r###" [