diff --git a/languages/tree-sitter-stack-graphs-python/.gitignore b/languages/tree-sitter-stack-graphs-python/.gitignore new file mode 100644 index 000000000..faf6459fb --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/.gitignore @@ -0,0 +1,3 @@ +*.html +/Cargo.lock +/target diff --git a/languages/tree-sitter-stack-graphs-python/CHANGELOG.md b/languages/tree-sitter-stack-graphs-python/CHANGELOG.md new file mode 100644 index 000000000..9e6604deb --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/CHANGELOG.md @@ -0,0 +1,6 @@ +# Changelog for tree-sitter-stack-graphs-python + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). diff --git a/languages/tree-sitter-stack-graphs-python/Cargo.toml b/languages/tree-sitter-stack-graphs-python/Cargo.toml new file mode 100644 index 000000000..4ac4c0633 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "tree-sitter-stack-graphs-python" +version = "0.1.0" +description = "Stack graphs definition for Python using tree-sitter-python" +readme = "README.md" +keywords = ["tree-sitter", "stack-graphs", "python"] +authors = [ + "GitHub ", +] +license = "MIT OR Apache-2.0" +edition = "2018" + +[[bin]] +name = "tree-sitter-stack-graphs-python" +path = "rust/bin.rs" +required-features = ["cli"] + +[lib] +path = "rust/lib.rs" +test = false + +[[test]] +name = "test" +path = "rust/test.rs" +harness = false + +[features] +cli = ["anyhow", "clap", "tree-sitter-stack-graphs/cli"] + +[dependencies] +anyhow = { version = "1.0", optional = true } +clap = { version = "4", optional = true, features = ["derive"] } +tree-sitter-stack-graphs = { version = "0.7", path = "../../tree-sitter-stack-graphs" } +tree-sitter-python = "=0.20.4" + +[dev-dependencies] +anyhow = "1.0" +tree-sitter-stack-graphs = { version = "0.7", path = "../../tree-sitter-stack-graphs", features = ["cli"] } diff --git a/languages/tree-sitter-stack-graphs-python/LICENSE b/languages/tree-sitter-stack-graphs-python/LICENSE new file mode 100644 index 000000000..e69de29bb diff --git a/languages/tree-sitter-stack-graphs-python/README.md b/languages/tree-sitter-stack-graphs-python/README.md new file mode 100644 index 000000000..b5fbc2f32 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/README.md @@ -0,0 +1,138 @@ +# tree-sitter-stack-graphs definition for Python + +This project defines tree-sitter-stack-graphs rules for Python using the [tree-sitter-python][] grammar. + +[tree-sitter-python]: https://crates.io/crates/tree-sitter-python + +- [API documentation](https://docs.rs/tree-sitter-stack-graphs-python/) +- [Release notes](https://github.com/github/stack-graphs/blob/main/languages/tree-sitter-stack-graphs-python/CHANGELOG.md) + +## Using the API + +To use this library, add the following to your `Cargo.toml`: + +```toml +[dependencies] +tree-sitter-stack-graphs-python = "0.1.0" +``` + +Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs-python/*/) for more details on how to use this library. + +## Using the Command-line Program + +The command-line program for `tree-sitter-stack-graphs-python` lets you do stack graph based analysis and lookup from the command line. + +The CLI can be run as follows: + +1. _(Installed)_ Install the CLI using Cargo as follows: + + ```sh + cargo install --features cli tree-sitter-stack-graphs-python + ``` + + After this, the CLI should be available as `tree-sitter-stack-graphs-python`. + +2. _(From source)_ Instead of installing the CLI, it can also be run directly from the crate directory, as a replacement for a `tree-sitter-stack-graphs-python` invocation, as follows: + + ```sh + cargo run --features cli -- + ``` + +The basic CLI workflow for the command-line program is to index source code and issue queries against the resulting database: + +1. Index a source folder as follows: + + ```sh + tree-sitter-stack-graphs-python index SOURCE_DIR + ``` + + _Indexing will skip any files that have already be indexed. To force a re-index, add the `-f` flag._ + + To check the status if a source folder, run: + + ```sh + tree-sitter-stack-graphs-python status SOURCE_DIR + ``` + + To clean the database and start with a clean slate, run: + + ```sh + tree-sitter-stack-graphs-python clean + ``` + + _Pass the `--delete` flag to not just empty the database, but also delete it. This is useful to resolve `unsupported database version` errors that may occur after a version update._ + +2. Run a query to find the definition(s) for a reference on a given line and column, run: + + ```sh + tree-sitter-stack-graphs-python query definition SOURCE_PATH:LINE:COLUMN + ``` + + Resulting definitions are printed, including a source line if the source file is available. + +Discover all available commands and flags by passing the `-h` flag to the CLI directly, or to any of the subcommands. + +## Development + +The project is written in Rust, and requires a recent version installed. Rust can be installed and updated using [rustup][]. + +[rustup]: https://rustup.rs/ + +The project is organized as follows: + +- The stack graph rules are defined in `src/stack-graphs.tsg`. +- Builtins sources and configuration are defined in `src/builtins.it` and `builtins.cfg` respectively. +- Tests are put into the `test` directory. + +### Running Tests + +Run the tests as follows: + +```sh +cargo test +``` + +The project consists of a library and a CLI. By default, running `cargo` only applies to the library. To run `cargo` commands on the CLI as well, add `--features cli` or `--all-features`. + +Run the CLI from source as follows: + +```sh +cargo run --features cli -- ARGS +``` + +Sources are formatted using the standard Rust formatted, which is applied by running: + +```sh +cargo fmt +``` + +### Writing TSG + +The stack graph rules are written in [tree-sitter-graph][]. Checkout the [examples][], +which contain self-contained TSG rules for specific language features. A VSCode +[extension][] is available that provides syntax highlighting for TSG files. + +[tree-sitter-graph]: https://github.com/tree-sitter/tree-sitter-graph +[examples]: https://github.com/github/stack-graphs/blob/main/tree-sitter-stack-graphs/examples/ +[extension]: https://marketplace.visualstudio.com/items?itemName=tree-sitter.tree-sitter-graph + +Parse and test a single file by executing the following commands: + +```sh +cargo run --features cli -- parse FILES... +cargo run --features cli -- test TESTFILES... +``` + +Generate a visualization to debug failing tests by passing the `-V` flag: + +```sh +cargo run --features cli -- test -V TESTFILES... +``` + +To generate the visualization regardless of test outcome, execute: + +```sh +cargo run --features cli -- test -V --output-mode=always TESTFILES... +``` + +Go to for links to examples and documentation. diff --git a/languages/tree-sitter-stack-graphs-python/rust/bin.rs b/languages/tree-sitter-stack-graphs-python/rust/bin.rs new file mode 100644 index 000000000..dc9a6772c --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/rust/bin.rs @@ -0,0 +1,32 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use anyhow::anyhow; +use clap::Parser; +use tree_sitter_stack_graphs::cli::database::default_user_database_path_for_crate; +use tree_sitter_stack_graphs::cli::provided_languages::Subcommands; +use tree_sitter_stack_graphs::NoCancellation; + +fn main() -> anyhow::Result<()> { + let lc = match tree_sitter_stack_graphs_python::try_language_configuration(&NoCancellation) { + Ok(lc) => lc, + Err(err) => { + eprintln!("{}", err.display_pretty()); + return Err(anyhow!("Language configuration error")); + } + }; + let cli = Cli::parse(); + let default_db_path = default_user_database_path_for_crate(env!("CARGO_PKG_NAME"))?; + cli.subcommand.run(default_db_path, vec![lc]) +} + +#[derive(Parser)] +#[clap(about, version)] +pub struct Cli { + #[clap(subcommand)] + subcommand: Subcommands, +} diff --git a/languages/tree-sitter-stack-graphs-python/rust/lib.rs b/languages/tree-sitter-stack-graphs-python/rust/lib.rs new file mode 100644 index 000000000..74176a475 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/rust/lib.rs @@ -0,0 +1,48 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use tree_sitter_stack_graphs::loader::LanguageConfiguration; +use tree_sitter_stack_graphs::loader::LoadError; +use tree_sitter_stack_graphs::CancellationFlag; + +/// The stack graphs tsg source for this language. +pub const STACK_GRAPHS_TSG_PATH: &str = "src/stack-graphs.tsg"; +/// The stack graphs tsg source for this language. +pub const STACK_GRAPHS_TSG_SOURCE: &str = include_str!("../src/stack-graphs.tsg"); + +/// The stack graphs builtins configuration for this language. +pub const STACK_GRAPHS_BUILTINS_CONFIG: &str = include_str!("../src/builtins.cfg"); +/// The stack graphs builtins path for this language +pub const STACK_GRAPHS_BUILTINS_PATH: &str = "src/builtins.py"; +/// The stack graphs builtins source for this language. +pub const STACK_GRAPHS_BUILTINS_SOURCE: &str = include_str!("../src/builtins.py"); + +/// The name of the file path global variable. +pub const FILE_PATH_VAR: &str = "FILE_PATH"; + +pub fn language_configuration(cancellation_flag: &dyn CancellationFlag) -> LanguageConfiguration { + try_language_configuration(cancellation_flag).unwrap_or_else(|err| panic!("{}", err)) +} + +pub fn try_language_configuration( + cancellation_flag: &dyn CancellationFlag, +) -> Result { + LanguageConfiguration::from_sources( + tree_sitter_python::language(), + Some(String::from("source.py")), + None, + vec![String::from("py")], + STACK_GRAPHS_TSG_PATH.into(), + STACK_GRAPHS_TSG_SOURCE, + Some(( + STACK_GRAPHS_BUILTINS_PATH.into(), + STACK_GRAPHS_BUILTINS_SOURCE, + )), + Some(STACK_GRAPHS_BUILTINS_CONFIG), + cancellation_flag, + ) +} diff --git a/languages/tree-sitter-stack-graphs-python/rust/test.rs b/languages/tree-sitter-stack-graphs-python/rust/test.rs new file mode 100644 index 000000000..c81edf0d4 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/rust/test.rs @@ -0,0 +1,23 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use anyhow::anyhow; +use std::path::PathBuf; +use tree_sitter_stack_graphs::ci::Tester; +use tree_sitter_stack_graphs::NoCancellation; + +fn main() -> anyhow::Result<()> { + let lc = match tree_sitter_stack_graphs_python::try_language_configuration(&NoCancellation) { + Ok(lc) => lc, + Err(err) => { + eprintln!("{}", err.display_pretty()); + return Err(anyhow!("Language configuration error")); + } + }; + let test_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test"); + Tester::new(vec![lc], vec![test_path]).run() +} diff --git a/languages/tree-sitter-stack-graphs-python/src/builtins.cfg b/languages/tree-sitter-stack-graphs-python/src/builtins.cfg new file mode 100644 index 000000000..d685061be --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/src/builtins.cfg @@ -0,0 +1 @@ +[globals] diff --git a/languages/tree-sitter-stack-graphs-python/src/builtins.py b/languages/tree-sitter-stack-graphs-python/src/builtins.py new file mode 100644 index 000000000..e69de29bb diff --git a/languages/tree-sitter-stack-graphs-python/src/stack-graphs.scm b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.scm new file mode 100644 index 000000000..380af7d95 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.scm @@ -0,0 +1,779 @@ +;; -*- coding: utf-8 -*- +;; ------------------------------------------------------------------------------------------------ +;; Copyright © 2023, stack-graphs authors. +;; Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +;; Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +;; ------------------------------------------------------------------------------------------------ + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; LEGACY DEFINITION! INCLUDED FOR REFERENCE ONLY! ;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; Modules and Imports +;--------------------- + +(module) @mod +{ + var module_def = @mod.file_def + attr module_def "no_span" + var module_ref = @mod.file_ref + attr module_ref "no_span" + var parent_module_def = module_def + var parent_module_ref = module_ref + var grandparent_module_ref = module_ref + + scan filepath { + "([^/]+)/" + { + edge module_def -> module_def.dot + edge module_def.dot -> module_def.next_def + edge module_ref.next_ref -> module_ref.dot + edge module_ref.dot -> module_ref + + attr module_def "pop" = $1 + attr module_def.dot "pop" = "." + attr module_ref "push" = $1 + attr module_ref.dot "push" = "." + + set grandparent_module_ref = parent_module_ref + set parent_module_def = module_def + set parent_module_ref = module_ref + set module_ref = module_ref.next_ref + set module_def = module_def.next_def + attr module_def "no_span" + attr module_ref "no_span" + } + + "__init__\\.py$" + { + attr parent_module_def "definition" + } + + "([^/]+)$" + { + edge module_def -> module_def.dot + edge module_def.dot -> module_def.next_def + + attr module_def "definition", "pop" = (replace $1 "\\.py" "") + attr module_def.dot "pop" = "." + + set module_def = module_def.next_def + attr module_def "no_span" + attr module_ref "no_span" + } + } + + edge root -> @mod.file_def + edge @mod.file_ref -> root + edge module_def -> @mod.after_scope + + edge @mod.before_scope -> @mod.global_dot + edge @mod.global -> root + attr @mod.global "push" = "" + + edge @mod.global_dot -> @mod.global + attr @mod.global_dot "push" = "." + + var @mod::parent_module = parent_module_ref + var @mod::grandparent_module = grandparent_module_ref + var @mod::bottom = @mod.after_scope + var @mod::global = @mod.global + var @mod::global_dot = @mod.global_dot +} + +(import_statement + name: (dotted_name + . (identifier) @root_name)) @stmt +{ + edge @stmt.after_scope -> @root_name.def, "precedence" = 1 + edge @root_name.ref -> root + attr @root_name.ref "push", "reference" + attr @root_name.def "pop", "definition" +} + +(import_statement + name: (aliased_import + (dotted_name . (identifier) @root_name))) @stmt +{ + edge @stmt.after_scope -> @root_name.def + edge @root_name.ref -> root + attr @root_name.ref "push", "reference" +} + +(import_from_statement + module_name: (dotted_name + . (identifier) @prefix_root_name)) @stmt +{ + edge @prefix_root_name.ref -> root + attr @prefix_root_name.ref "push", "reference" +} + +(import_from_statement + name: (dotted_name + . (identifier) @import_root_name)) @stmt +{ + edge @stmt.after_scope -> @import_root_name.def, "precedence" = 1 + edge @import_root_name.ref -> @import_root_name.ref_dot + attr @import_root_name.def "pop", "definition" + attr @import_root_name.ref "push", "reference" + attr @import_root_name.ref_dot "push" = "." +} + +(import_from_statement + name: (aliased_import + (dotted_name + . (identifier) @import_root_name))) @stmt +{ + edge @import_root_name.ref -> @import_root_name.ref_dot + attr @import_root_name.ref "push", "reference" + attr @import_root_name.ref_dot "push" = "." +} + +(import_from_statement + module_name: [ + (dotted_name (identifier) @prefix_leaf_name .) + (relative_import (dotted_name (identifier) @prefix_leaf_name .)) + (relative_import (import_prefix) @prefix_leaf_name .) + ] + name: [ + (dotted_name + . (identifier) @import_root_name) + (aliased_import + (dotted_name + . (identifier) @import_root_name)) + ]) +{ + edge @import_root_name.ref_dot -> @prefix_leaf_name.ref +} + +[ + (import_from_statement + (aliased_import + name: (dotted_name (identifier) @name .) + alias: (identifier) @alias)) + (import_statement + (aliased_import + name: (dotted_name (identifier) @name .) + alias: (identifier) @alias)) +] @stmt +{ + edge @stmt.after_scope -> @alias + edge @alias -> @name.ref + attr @alias "pop", "definition" +} + +[ + (import_statement + name: (dotted_name + (identifier) @leaf_name .)) + (import_from_statement + name: (dotted_name + (identifier) @leaf_name .)) +] +{ + attr @leaf_name.def "pop", "definition" + attr @leaf_name.ref "push", "reference" + edge @leaf_name.def -> @leaf_name.ref +} + +(relative_import + (import_prefix) @prefix + (#eq? @prefix ".")) @import +{ + edge @prefix.ref -> @import::parent_module +} + +(relative_import + (import_prefix) @prefix + (#eq? @prefix "..")) @import +{ + edge @prefix.ref -> @import::grandparent_module +} + +(relative_import + (import_prefix) @prefix + (dotted_name + . (identifier) @name)) +{ + attr @name.ref "push", "reference" + attr @name.ref_dot "push" = "." + edge @name.ref -> @name.ref_dot + edge @name.ref_dot -> @prefix.ref +} + +[ + (import_from_statement + module_name: (relative_import + (dotted_name + (identifier) @parent_name + . + (identifier) @child_name))) + (import_from_statement + module_name: (dotted_name + (identifier) @parent_name + . + (identifier) @child_name)) +] +{ + attr @child_name.ref "push", "reference" + attr @child_name.ref_dot "push" = "." + edge @child_name.ref -> @child_name.ref_dot + edge @child_name.ref_dot -> @parent_name.ref +} + +(import_from_statement + module_name: (dotted_name + . (identifier) @root_name)) +{ + attr @root_name.ref "push", "reference" + edge @root_name.ref -> root +} + +(import_from_statement + module_name: (dotted_name + (identifier) @leaf_name .) + (wildcard_import) @star) @stmt +{ + edge @stmt.after_scope -> @star.ref_dot, "precedence" = 1 + edge @star.ref_dot -> @leaf_name.ref + attr @star.ref_dot "push" = "." +} + +[ + (import_statement + name: (dotted_name + (identifier) @parent_name + . + (identifier) @child_name)) + (import_from_statement + name: (dotted_name + (identifier) @parent_name + . + (identifier) @child_name)) + (import_from_statement + name: (aliased_import + name: (dotted_name + (identifier) @parent_name + . + (identifier) @child_name))) +] +{ + edge @child_name.ref -> @child_name.ref_dot + edge @child_name.ref_dot -> @parent_name.ref + edge @parent_name.def -> @parent_name.def_dot + edge @parent_name.def_dot -> @child_name.def + attr @child_name.def "pop", "definition" + attr @child_name.ref "push","reference" + attr @parent_name.def_dot "pop" = "." + attr @child_name.ref_dot "push" = "." +} + +;-------- +; Scopes +;-------- + +[ + (module (_) @last_stmt .) + (block (_) @last_stmt .) +] @block +{ + edge @block.after_scope -> @last_stmt.after_scope +} + +[ + (module (_) @stmt1 . (_) @stmt2) + (block (_) @stmt1 . (_) @stmt2) +] +{ + edge @stmt2.before_scope -> @stmt1.after_scope +} + +[ + (module (_) @stmt) + (block (_) @stmt) +] +{ + edge @stmt.after_scope -> @stmt.before_scope + let @stmt::local_scope = @stmt.before_scope +} + +[ + (block . (_) @stmt) + (module . (_) @stmt) +] @block +{ + edge @stmt.before_scope -> @block.before_scope +} + +(block (_) @stmt . ) @block +{ + edge @block.after_scope -> @stmt.after_scope +} + +(function_definition (block) @block) +{ + edge @block.before_scope -> @block::local_scope +} + +[ + (while_statement (block) @block) + (if_statement (block) @block) + (with_statement (block) @block) + (try_statement (block) @block) + (for_statement (block) @block) + (_ [ + (else_clause (block) @block) + (elif_clause (block) @block) + (except_clause (block) @block) + (finally_clause (block) @block) + ]) +] @stmt +{ + edge @block.before_scope -> @block::local_scope + edge @stmt.after_scope -> @block.after_scope +} + +(match_statement (case_clause) @block) @stmt +{ + let @block::local_scope = @block.before_scope + edge @block.before_scope -> @stmt.before_scope + edge @stmt.after_scope -> @block.after_scope +} + +[ + (for_statement) + (while_statement) +] @stmt +{ + edge @stmt.before_scope -> @stmt.after_scope +} + +;------------- +; Definitions +;------------- + +[ + (assignment + left: (_) @pattern + right: (_) @value) + (with_item + value: + (as_pattern + (_) @value + alias: (as_pattern_target (_) @pattern))) +] +{ + edge @pattern.input -> @value.output +} + +(function_definition + name: (identifier) @name + parameters: (parameters) @params + body: (block) @body) @func +{ + attr @name "definiens" = @func + edge @func.after_scope -> @name + edge @name -> @func.call + edge @func.call -> @func.return_value + edge @body.before_scope -> @params.after_scope + edge @body.before_scope -> @func.drop_scope + edge @func.drop_scope -> @func::bottom + attr @func.drop_scope "drop" + attr @name "pop", "definition" + attr @func.call "pop" = "()", "pop-scope" + attr @params.before_scope "jump-to" + attr @func.return_value "endpoint" + let @func::function_returns = @func.return_value + + ; Prevent functions defined inside of method bodies from being treated like methods + let @body::class_self_scope = nil + let @body::class_member_attr_scope = nil +} + +;; +;; BEGIN BIG GNARLY DISJUNCTION +;; +;; The following pair of rules is intended to capture the following behavior: +;; +;; If a function definition is used to define a method, by being inside a class +;; definition, then we make its syntax type `method`. Otherwise, we make it's +;; syntax type `function`. Unfortunately, because of the limitations on negation +;; and binding in tree sitter queries, we cannot negate `class_definition` or +;; similar things directly. Instead, we have to manually push the negation down +;; to form the finite disjunction it corresponds to. +;; + +[ + (class_definition (block (decorated_definition (function_definition name: (_)@name)))) + (class_definition (block (function_definition name: (_)@name))) +] +{ + attr @name "syntax_type" = "method" +} + +[ + (module (decorated_definition (function_definition name: (_)@name))) + (module (function_definition name: (_)@name)) + + (if_statement (block (decorated_definition (function_definition name: (_)@name)))) + (if_statement (block (function_definition name: (_)@name))) + + (elif_clause (block (decorated_definition (function_definition name: (_)@name)))) + (elif_clause (block (function_definition name: (_)@name))) + + (else_clause (block (decorated_definition (function_definition name: (_)@name)))) + (else_clause (block (function_definition name: (_)@name))) + + (case_clause (block (decorated_definition (function_definition name: (_)@name)))) + (case_clause (block (function_definition name: (_)@name))) + + (for_statement (block (decorated_definition (function_definition name: (_)@name)))) + (for_statement (block (function_definition name: (_)@name))) + + (while_statement (block (decorated_definition (function_definition name: (_)@name)))) + (while_statement (block (function_definition name: (_)@name))) + + (try_statement (block (decorated_definition (function_definition name: (_)@name)))) + (try_statement (block (function_definition name: (_)@name))) + + (except_clause (block (decorated_definition (function_definition name: (_)@name)))) + (except_clause (block (function_definition name: (_)@name))) + + (finally_clause (block (decorated_definition (function_definition name: (_)@name)))) + (finally_clause (block (function_definition name: (_)@name))) + + (with_statement (block (decorated_definition (function_definition name: (_)@name)))) + (with_statement (block (function_definition name: (_)@name))) + + (function_definition (block (decorated_definition (function_definition name: (_)@name)))) + (function_definition (block (function_definition name: (_)@name))) +] +{ + attr @name "syntax_type" = "function" +} + +;; +;; END BIG GNARLY DISJUNCTION +;; + +(function_definition + parameters: (parameters + . (identifier) @param) + body: (block) @body) +{ + edge @param.input -> @param::class_self_scope + edge @param::class_member_attr_scope -> @param.output + edge @param.output -> @body.after_scope + attr @param.output "push" +} + +(parameter/identifier) @param +{ + attr @param.input "definition", "pop" + attr @param.param_name "push" + edge @param.input -> @param.param_index + edge @param.input -> @param.param_name +} + +[ + (parameter/default_parameter + name: (identifier) @name + value: (_) @value) @param + (parameter/typed_default_parameter + name: (_) @name + value: (_) @value) @param +] +{ + attr @name "definition", "pop" + attr @param.param_name "push" = @name + edge @name -> @param.param_name + edge @name -> @param.param_index + edge @param.input -> @name + edge @name -> @value.output +} + +[ + (parameter/typed_parameter + . (_) @name) @param + (parameter/list_splat_pattern + (_) @name) @param + (parameter/dictionary_splat_pattern + (_) @name) @param +] +{ + attr @name "definition", "pop" + attr @param.param_name "push" = @name + edge @name -> @param.param_name + edge @name -> @param.param_index + edge @param.input -> @name +} + +[ + (pattern_list (_) @pattern) + (tuple_pattern (_) @pattern) +] @list +{ + let statement_scope = @list::local_scope + let @pattern::local_scope = @pattern.pattern_before_scope + edge statement_scope -> @pattern::local_scope, "precedence" = (+ 1 (child-index @pattern)) + + edge @pattern.pattern_index -> @list.input + edge @pattern.input -> @pattern.pattern_index + attr @pattern.pattern_index "push" = (child-index @pattern) +} + +(parameters + (_) @param) @params +{ + attr @param.param_index "push" = (child-index @param) + edge @param.param_index -> @params.before_scope + edge @params.after_scope -> @param.input + edge @param.param_name -> @params.before_scope +} + +(return_statement (_) @expr) @stmt +{ + edge @stmt::function_returns -> @expr.output +} + +(class_definition + name: (identifier) @name) @class +{ + attr @name "definiens" = @class + attr @name "syntax_type" = "class" + edge @class.parent_scope -> @class::class_parent_scope + edge @class.parent_scope -> @class::local_scope + edge @class.after_scope -> @name + edge @name -> @class.call + edge @name -> @class.dot + edge @class.dot -> @class.members + edge @class.call -> @class.call_drop + edge @class.call_drop -> @class.self_scope + edge @class.self_scope -> @class.super_scope + edge @class.self_scope -> @class.self_dot + edge @class.self_dot -> @class.members + edge @class.members -> @class.member_attrs + attr @class.call "pop" = "()", "pop-scope" + attr @class.call_drop "drop" + attr @class.dot "pop" = "." + attr @class.self_dot "pop" = "." + attr @name "pop", "definition" + attr @class.member_attrs "push" = "." + attr @class.self_scope "endpoint" + let @class::super_scope = @class.super_scope + let @class::class_parent_scope = @class.parent_scope + let @class::class_self_scope = @class.call_drop + let @class::class_member_attr_scope = @class.member_attrs +} + +(class_definition + body: (block + (_) @last_stmt .) @body) @class +{ + edge @class.members -> @last_stmt.after_scope +} + +(class_definition + superclasses: (argument_list + (_) @superclass)) @class +{ + edge @class.super_scope -> @superclass.output +} + +(decorated_definition + definition: (_) @def) @stmt +{ + edge @def.before_scope -> @stmt.before_scope + edge @stmt.after_scope -> @def.after_scope +} + +(case_clause + pattern: (_) @pattern + consequence: (_) @consequence) @clause +{ + edge @consequence.before_scope -> @pattern.new_bindings + edge @consequence.before_scope -> @clause.before_scope + edge @clause.after_scope -> @consequence.after_scope +} + +;------------- +; Expressions +;------------- + +(call + function: (_) @fn + arguments: (argument_list) @args) @call +{ + edge @call.output -> @call.output_args + edge @call.output_args -> @fn.output + attr @call.output_args "push" = "()", "push-scope" = @args +} + +(call + function: (attribute + object: (_) @receiver) + arguments: (argument_list + (expression) @arg) @args) +{ + edge @args -> @arg.arg_index + edge @receiver -> @receiver.arg_index + + attr @receiver.arg_index "pop" = "0" + edge @receiver.arg_index -> @receiver.output + + attr @arg.arg_index "pop" = (+ 1 (child-index @arg)) + edge @arg.arg_index -> @arg.output +} + +(call + arguments: (argument_list + (keyword_argument + name: (identifier) @name + value: (_) @val) @arg) @args) @call +{ + edge @args -> @arg.arg_name + attr @arg.arg_name "pop" = @name + edge @arg.arg_name -> @val.output +} + +(argument_list + (expression) @arg) @args +{ + edge @args -> @arg.arg_index + attr @arg.arg_index "pop" = (child-index @arg) + edge @arg.arg_index -> @arg.output +} + +( + (call + function: (identifier) @fn-name) @call + (#eq? @fn-name "super") +) +{ + edge @call.output -> @call::super_scope +} + +[ + (tuple (_) @element) + (expression_list (_) @element) +] @tuple +{ + edge @tuple.output -> @element.el_index + attr @element.el_index "pop" = (child-index @element) + edge @element.el_index -> @element.output + + edge @tuple.new_bindings -> @element.new_bindings +} + +(attribute + object: (_) @object + attribute: (identifier) @name) @expr +{ + edge @expr.output -> @name.output + edge @name.output -> @expr.output_dot + edge @expr.output_dot -> @object.output + edge @object.input -> @expr.input_dot + edge @expr.input_dot -> @name.input + edge @name.input -> @expr.input + attr @expr.output_dot "push" = "." + attr @expr.input_dot "pop" = "." + attr @name.input "pop" + attr @name.output "push" +} + +(pattern/attribute + attribute: (identifier) @name) +{ + attr @name.input "definition" +} + +(primary_expression/attribute + attribute: (identifier) @name) +{ + attr @name.output "reference" +} + +(primary_expression/identifier) @id +{ + edge @id.output -> @id::local_scope + edge @id.output -> @id::class_parent_scope + edge @id::local_scope -> @id.input + attr @id.input "pop" + attr @id.output "push", "reference" + + attr @id.new_binding_pop "pop", "definition" + edge @id.new_bindings -> @id.new_binding_pop +} + +(pattern/identifier) @id +{ + edge @id.output -> @id::local_scope + edge @id.output -> @id::class_parent_scope + edge @id::local_scope -> @id.input, "precedence" = 1 + attr @id.input "pop", "definition" + attr @id.output "push" + + attr @id.new_binding_pop "pop", "definition" + edge @id.new_bindings -> @id.new_binding_pop +} + +(as_pattern + (expression) @value + alias: (as_pattern_target (primary_expression/identifier) @id)) @as_pattern +{ + edge @id.output -> @id::local_scope + edge @id.output -> @id::class_parent_scope + edge @id::local_scope -> @id.input, "precedence" = 1 + attr @id.input "pop", "definition" + attr @id.output "push" + + edge @as_pattern.new_bindings -> @value.new_bindings + edge @as_pattern.new_bindings -> @id.new_bindings +} + +(list) @list +{ + edge @list.output -> @list.called + edge @list.called -> @list::global_dot + attr @list.called "push" = "list" +} + +(list (_) @el) @list +{ + edge @list.new_bindings -> @el.new_bindings +} + +(dictionary (pair) @pair) @dict +{ + edge @dict.new_bindings -> @pair.new_bindings +} + +(pair + value: (_) @value) @pair +{ + edge @pair.new_bindings -> @value.new_bindings +} + +(set (_) @el) @set +{ + edge @set.new_bindings -> @el.new_bindings +} + +(list_splat (_) @splatted) @splat +{ +attr @splat.new_bindings_pop "pop" = @splatted, "definition" +edge @splat.new_bindings -> @splat.new_bindings_pop +} + +(binary_operator + (_) @left + (_) @right) @binop +{ + edge @binop.new_bindings -> @left.new_bindings + edge @binop.new_bindings -> @right.new_bindings +} + +(case_pattern (_) @expr) @pat +{ + edge @pat.new_bindings -> @expr.new_bindings +} diff --git a/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg new file mode 100644 index 000000000..7cddd6783 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg @@ -0,0 +1,1346 @@ +;; -*- coding: utf-8 -*- +;; ------------------------------------------------------------------------------------------------ +;; Copyright © 2023, stack-graphs authors. +;; Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +;; Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +;; ------------------------------------------------------------------------------------------------ + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Stack graphs definition for Python +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Global Variables +;; ^^^^^^^^^^^^^^^^ + +global FILE_PATH +global ROOT_NODE +global JUMP_TO_SCOPE_NODE + +;; Attribute Shorthands +;; ^^^^^^^^^^^^^^^^^^^^ + +attribute node_definition = node => type = "pop_symbol", node_symbol = node, is_definition +attribute node_reference = node => type = "push_symbol", node_symbol = node, is_reference +attribute pop_node = node => type = "pop_symbol", node_symbol = node +attribute pop_scoped_node = node => type = "pop_scoped_symbol", node_symbol = node +attribute pop_scoped_symbol = symbol => type = "pop_scoped_symbol", symbol = symbol +attribute pop_symbol = symbol => type = "pop_symbol", symbol = symbol +attribute push_node = node => type = "push_symbol", node_symbol = node +attribute push_scoped_node = node => type = "push_scoped_symbol", node_symbol = node +attribute push_scoped_symbol = symbol => type = "push_scoped_symbol", symbol = symbol +attribute push_symbol = symbol => type = "push_symbol", symbol = symbol +attribute scoped_node_definition = node => type = "pop_scoped_symbol", node_symbol = node, is_definition +attribute scoped_node_reference = node => type = "push_scoped_symbol", node_symbol = node, is_reference +attribute symbol_definition = symbol => type = "pop_symbol", symbol = symbol, is_definition +attribute symbol_reference = symbol => type = "push_symbol", symbol = symbol, is_reference + +attribute node_symbol = node => symbol = (source-text node), source_node = node + +;; Nodes +;; ^^^^^ + +(module) @node { + node @node.after_scope + node @node.before_scope +} + +[ + ; _statement + ; _simple_statement + (future_import_statement) + (import_statement) + (import_from_statement) + (print_statement) + (assert_statement) + (expression_statement) + (return_statement) + (delete_statement) + (raise_statement) + (pass_statement) + (break_statement) + (continue_statement) + (global_statement) + (nonlocal_statement) + (exec_statement) + (type_alias_statement) + ; _compound_statement + (if_statement) + (for_statement) + (while_statement) + (try_statement) + (with_statement) + (function_definition) + (class_definition) + (decorated_definition) + (match_statement) + ; block + (block) + ; statement clauses + (if_clause) + (elif_clause) + (else_clause) + (except_group_clause) + (except_clause) + (finally_clause) + (with_clause) + (case_clause) +] @node { + node @node.after_scope + node @node.before_scope +} + +[ + (parameters) + (lambda_parameters) +] @node { + node @node.after_scope + node @node.before_scope +} + +[ + (identifier) +] @node { + node @node.def + node @node.def_dot + node @node.ref + node @node.ref_dot +} + +[ + (dotted_name) + (aliased_import) + (relative_import) + (wildcard_import) + (import_prefix) +] @node { + node @node.after_scope + node @node.before_scope + node @node.def + node @node.ref +} + +[ + ; expressions + (comparison_operator) + (not_operator) + (boolean_operator) + (lambda) + ;(primary_expression) ; unfolded below + (conditional_expression) + (named_expression) + (as_pattern) + ; primary_expression + (await) + (binary_operator) + (identifier) + ;(keyword_identifier) ; invalid query pattern? + (string) + (concatenated_string) + (integer) + (float) + (true) + (false) + (none) + (unary_operator) + (attribute) + (subscript) + (call) + (list) + (list_comprehension) + (dictionary) + (dictionary_comprehension) + (set) + (set_comprehension) + (tuple) + (parenthesized_expression) + (generator_expression) + (ellipsis) + (list_splat) + + ; expression list + (expression_list) + + ; pattern + (pattern/identifier) + ;(keyword_identifier) ; invalid query pattern? + ;(subscript) + ;(attribute) + (list_splat_pattern) + (tuple_pattern) + (list_pattern) + ; _simple_patterns + (class_pattern) + (splat_pattern) + (union_pattern) + ;(list_pattern) ; already in pattern + ;(tuple_pattern) ; already in pattern + (dict_pattern) + ;(string) ; already in primary_expression + ;(concatenated_string) ; already in primary_expression + ;(true) ; already in primary_expression + ;(false) ; already in primary_expression + ;(none) ; already in primary_expression + ;(integer) ; already in primary_expression + ;(float) ; already in primary_expression + (complex_pattern) + (dotted_name) + ; _as_attern + (as_pattern) + ; keyword pattern + (keyword_pattern) + ; case pattern + (case_pattern) + ; with item + (with_item) + + ; pattern list + (pattern_list) + + ; parameter + ;(identifier) ; already in expressions + (typed_parameter) + (default_parameter) + (typed_default_parameter) + ;(list_splat_pattern) ; already in patterns + ;(tuple_pattern) ; already in patterns + (keyword_separator) + (positional_separator) + (dictionary_splat_pattern) + + ; parameters + (parameters) +] @node { + node @node.input + node @node.new_bindings + node @node.output +} + +(comment) @node { + node @node.after_scope + node @node.before_scope + node @node.def + node @node.def_dot + node @node.input + node @node.new_bindings + node @node.output + node @node.ref + node @node.ref_dot +} + +;; Inherited Variables +;; ^^^^^^^^^^^^^^^^^^^ + +inherit .bottom +inherit .class_member_attr_scope +inherit .class_parent_scope +inherit .class_self_scope +inherit .class_super_scope +inherit .function_returns +inherit .global +inherit .global_dot +inherit .grandparent_module +inherit .local_scope +inherit .parent_module + +;; +;; # # +;; ## ## #### ##### # # # ###### #### +;; # # # # # # # # # # # # # +;; # # # # # # # # # # ##### #### +;; # # # # # # # # # # # +;; # # # # # # # # # # # # +;; # # #### ##### #### ###### ###### #### +;; +;; Modules + +(module) @mod +{ + node mod_file_def + node mod_file_ref + + var module_def = mod_file_def + + node parent_module_def_node + var parent_module_def = parent_module_def_node + + var module_ref = mod_file_ref + + node parent_module_ref_node + var parent_module_ref = parent_module_ref_node + + node grandparent_module_ref_node + var grandparent_module_ref = grandparent_module_ref_node + + scan FILE_PATH { + "([^/]+)/" + { + node def_dot + attr (def_dot) pop_symbol = "." + node next_def + ; + edge module_def -> def_dot + edge def_dot -> next_def + ; + attr (module_def) pop_symbol = $1 + ; + set parent_module_def = module_def + set module_def = next_def + + node ref_dot + attr (ref_dot) push_symbol = "." + node next_ref + ; + edge next_ref -> ref_dot + edge ref_dot -> module_ref + ; + attr (module_ref) push_symbol = $1 + ; + set grandparent_module_ref = parent_module_ref + set parent_module_ref = module_ref + set module_ref = next_ref + } + + "__init__\.py$" + { + attr (parent_module_def) is_definition, source_node = @mod, empty_source_span + } + + "([^/]+)\.py$" + { + node def_dot + attr (def_dot) pop_symbol = "." + node next_def + ; + edge module_def -> def_dot + edge def_dot -> next_def + ; + attr (module_def) pop_symbol = $1, is_definition, source_node = @mod, empty_source_span + ; + set module_def = next_def + } + } + + edge ROOT_NODE -> mod_file_def + edge mod_file_ref -> ROOT_NODE + edge module_def -> @mod.after_scope + + node global + node global_dot + + edge @mod.before_scope -> global_dot + edge global -> ROOT_NODE + attr (global) push_symbol = "" + + edge global_dot -> global + attr (global_dot) push_symbol = "." + + let @mod.parent_module = parent_module_ref + let @mod.grandparent_module = grandparent_module_ref + let @mod.bottom = @mod.after_scope + let @mod.global = global + let @mod.global_dot = global_dot + + ;; add a dummy nodes for inherited variables + node @mod.class_member_attr_scope + node @mod.class_parent_scope + node @mod.class_self_scope + node @mod.class_super_scope + node @mod.function_returns + node @mod.local_scope +} + +;; +;; ### +;; # # # ##### #### ##### ##### #### +;; # ## ## # # # # # # # # +;; # # ## # # # # # # # # #### +;; # # # ##### # # ##### # # +;; # # # # # # # # # # # +;; ### # # # #### # # # #### +;; +;; Imports + +;; Import References +;; ^^^^^^^^^^^^^^^^^ + +;;;; Dotted Names +;; +;; (dotted_name).ref node to connect to to use the reference +;; (dotted_name).before_scope node to connect from to ensure the reference resolves + +;; all names are references +[ + (import_statement name: (dotted_name (identifier) @name)) + (future_import_statement name: (dotted_name (identifier) @name)) + (import_from_statement name: (dotted_name (identifier) @name)) + (import_statement name: (aliased_import name: (dotted_name (identifier) @name))) + (future_import_statement name: (aliased_import name: (dotted_name (identifier) @name))) + (import_from_statement name: (aliased_import name: (dotted_name (identifier) @name))) + (import_from_statement module_name: (dotted_name (identifier) @name)) + (import_from_statement module_name: (relative_import (dotted_name (identifier) @name))) +] { + attr (@name.ref) node_reference = @name +} + +;; references are chained +[ + (import_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (future_import_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (import_from_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (import_statement name: (aliased_import name: (dotted_name (identifier) @left . (identifier) @right))) + (future_import_statement name: (aliased_import name: (dotted_name (identifier) @left . (identifier) @right))) + (import_from_statement name: (aliased_import name: (dotted_name (identifier) @left . (identifier) @right))) + (import_from_statement module_name: (dotted_name (identifier) @left . (identifier) @right)) + (import_from_statement module_name: (relative_import (dotted_name (identifier) @left . (identifier) @right))) +] { + node push_dot + attr (push_dot) push_symbol = "." + + edge @right.ref -> push_dot + edge push_dot -> @left.ref +} + +;; lookup first reference +[ + (import_statement name: (dotted_name . (identifier) @first) @dotted) + (future_import_statement name: (dotted_name . (identifier) @first) @dotted) + (import_from_statement name: (dotted_name . (identifier) @first) @dotted) + (import_statement name: (aliased_import name: (dotted_name . (identifier) @first) @dotted)) + (future_import_statement name: (aliased_import name: (dotted_name . (identifier) @first) @dotted)) + (import_from_statement name: (aliased_import name: (dotted_name . (identifier) @first) @dotted)) + (import_from_statement module_name: (dotted_name . (identifier) @first) @dotted) + (import_from_statement module_name: (relative_import (dotted_name . (identifier) @first) @dotted)) +] { + edge @first.ref -> @dotted.before_scope +} + +;; expose last reference +[ + (import_statement name: (dotted_name (identifier) @last .) @dotted) + (future_import_statement name: (dotted_name (identifier) @last .) @dotted) + (import_from_statement name: (dotted_name (identifier) @last .) @dotted) + (import_statement name: (aliased_import name: (dotted_name (identifier) @last .) @dotted)) + (future_import_statement name: (aliased_import name: (dotted_name (identifier) @last .) @dotted)) + (import_from_statement name: (aliased_import name: (dotted_name (identifier) @last .) @dotted)) + (import_from_statement module_name: (dotted_name (identifier) @last .) @dotted) + (import_from_statement module_name: (relative_import (dotted_name (identifier) @last .) @dotted)) +] { + edge @dotted.ref -> @last.ref +} + +;;;; Aliased Import +;; +;; An aliased import behaves like its wrapped dotted_name as a reference +;; +;; (aliased_import).ref node to connect to, to use the reference +;; (aliased_import).before_scope node to connect from to ensure the reference resolves + +(aliased_import name: (dotted_name) @dotted) @aliased { + edge @aliased.ref -> @dotted.ref + edge @dotted.before_scope -> @aliased.before_scope +} + +;;;; Relative Import +;; +;; A relative import behaves like its wrapped dotted_name as a reference +;; +;; (relative_import).ref node to connect to, to use the reference +;; (relative_import).before_scope node to connect from to ensure the reference resolves + +(relative_import (import_prefix) . (dotted_name) @dotted) @relative { + edge @relative.ref -> @dotted.ref + + node push_dot + attr (push_dot) push_symbol = "." + + edge @dotted.before_scope -> push_dot + edge push_dot -> @relative.before_scope +} + +(relative_import (import_prefix) .) @relative { + edge @relative.ref -> @relative.before_scope +} + +;;;; Wildcard Import +;; +;; A wildcard import simply passes through +;; +;; (wildcard_import).ref node to connect to, to use the reference +;; (wildcard_import).before_scope node to connect from to ensure the reference resolves + +(wildcard_import) @wildcard { + edge @wildcard.ref -> @wildcard.before_scope +} + +;;;; Import from +;; +;; The imported references are resolved in the from module + +[ + (import_from_statement module_name: (_) @left name: (_) @right) + (import_from_statement module_name: (_) @left (wildcard_import) @right) +] { + node push_dot + attr (push_dot) push_symbol = "." + + edge @right.before_scope -> push_dot + edge push_dot -> @left.ref +} + +;;;; Non-relative Imports +;; +;; Non-relative imports are resolved in the root scope + +[ + (import_statement name: (_) @name) + (import_from_statement module_name: (dotted_name) @name) +] { + edge @name.before_scope -> ROOT_NODE +} + +;;;; Relative Imports +;; +;; Relative imports are resolved in scopes related to the current module + +;; . imports resolve in parent module scope +(import_from_statement module_name: (relative_import (import_prefix) @prefix (#eq? @prefix ".")) @relative) { + edge @relative.before_scope -> @prefix.parent_module +} + +;; .. imports resolve in grandparent module scope +(import_from_statement module_name: (relative_import (import_prefix) @prefix (#eq? @prefix "..")) @relative) { + edge @relative.before_scope -> @prefix.grandparent_module +} + +;;;; Future Imports +;; +;; We don't know the future, so we cannot connect references of future imports anywhere. Maybe one day? + +;; Import Definitions +;; ^^^^^^^^^^^^^^^^^^ + +;;;; Dotted Names & Aliased Imports +;; +;; (dotted_name).after_scope node to connect to, to expose the definition +;; (dotted_name).def node to connect from, to give definition content + +;; unaliased names and aliases are definitions +[ + (import_statement name: (dotted_name (identifier) @name)) + (future_import_statement name: (dotted_name (identifier) @name)) + (import_from_statement name: (dotted_name (identifier) @name)) + (import_statement name: (aliased_import alias: (identifier) @name)) + (future_import_statement name: (aliased_import alias: (identifier) @name)) + (import_from_statement name: (aliased_import alias: (identifier) @name)) +] { + attr (@name.def) node_definition = @name +} + +;; definitions are chained +[ + (import_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (future_import_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (import_from_statement name: (dotted_name (identifier) @left . (identifier) @right)) +] { + node pop_dot + attr (pop_dot) pop_symbol = "." + + edge @left.def -> pop_dot + edge pop_dot -> @right.def +} + +;; connect last definition +[ + (import_statement name: (dotted_name (identifier) @last .) @outer) + (future_import_statement name: (dotted_name (identifier) @last .) @outer) + (import_from_statement name: (dotted_name (identifier) @last .) @outer) + (import_statement name: (aliased_import alias: (identifier) @last ) @outer) + (future_import_statement name: (aliased_import alias: (identifier) @last ) @outer) + (import_from_statement name: (aliased_import alias: (identifier) @last ) @outer) +] { + edge @last.def -> @outer.def +} + +;; expose first definition +[ + (import_statement name: (dotted_name . (identifier) @first) @outer) + (future_import_statement name: (dotted_name . (identifier) @first) @outer) + (import_from_statement name: (dotted_name . (identifier) @first) @outer) + (import_statement name: (aliased_import alias: (identifier) @first ) @outer) + (future_import_statement name: (aliased_import alias: (identifier) @first ) @outer) + (import_from_statement name: (aliased_import alias: (identifier) @first ) @outer) +] { + edge @outer.after_scope -> @first.def +} + +;;;; Wildcard Import +;; +;; Wildcard imports simply pass through + +(wildcard_import) @wildcard { + edge @wildcard.after_scope -> @wildcard.def +} + +;;;; Import Definitions -> References +;; +;; The definitions introduced by imports are connected to the corresponding references + +[ + (import_statement name: (_) @name) + (future_import_statement name: (_) @name) + (import_from_statement name: (_) @name) + (import_from_statement (wildcard_import) @name) +] { + edge @name.def -> @name.ref +} + +;;;; Imports +;; +;; The definitions introduced by imports are visible after the import statement + +[ + (import_statement name: (_) @name) + (future_import_statement name: (_) @name) + (import_from_statement name: (_) @name) + (import_from_statement (wildcard_import) @name) +] @stmt { + edge @stmt.after_scope -> @name.after_scope + attr (@stmt.after_scope -> @name.after_scope) precedence = 1 +} + +;; +;; ###### +;; # # # #### #### # # #### +;; # # # # # # # # # # +;; ###### # # # # #### #### +;; # # # # # # # # # +;; # # # # # # # # # # # +;; ###### ###### #### #### # # #### +;; +;; Blocks + +[ + (module (_) @last_stmt .) + (block (_) @last_stmt .) +] @block +{ + edge @block.after_scope -> @last_stmt.after_scope +} + +[ + (module (_) @stmt1 . (_) @stmt2) + (block (_) @stmt1 . (_) @stmt2) +] +{ + edge @stmt2.before_scope -> @stmt1.after_scope +} + +[ + (module (_) @stmt) + (block (_) @stmt) +] +{ + edge @stmt.after_scope -> @stmt.before_scope + let @stmt.local_scope = @stmt.before_scope +} + +[ + (block . (_) @stmt) + (module . (_) @stmt) +] @block +{ + edge @stmt.before_scope -> @block.before_scope +} + +(function_definition (block) @block) +{ + edge @block.before_scope -> @block.local_scope +} + +[ + (while_statement (block) @block) + (if_statement (block) @block) + (with_statement (block) @block) + (try_statement (block) @block) + (for_statement (block) @block) + (_ [ + (else_clause (block) @block) + (elif_clause (block) @block) + (except_clause (block) @block) + (finally_clause (block) @block) + ]) +] @stmt +{ + edge @block.before_scope -> @block.local_scope + edge @stmt.after_scope -> @block.after_scope +} + +(match_statement body: (_) @block) @stmt +{ + let @block.local_scope = @block.before_scope + edge @block.before_scope -> @stmt.before_scope + edge @stmt.after_scope -> @block.after_scope +} + +[ + (for_statement) + (while_statement) +] @stmt +{ + edge @stmt.before_scope -> @stmt.after_scope +} + +;; +;; ##### +;; # # ##### ## ##### ###### # # ###### # # ##### #### +;; # # # # # # ## ## # ## # # # +;; ##### # # # # ##### # ## # ##### # # # # #### +;; # # ###### # # # # # # # # # # +;; # # # # # # # # # # # ## # # # +;; ##### # # # # ###### # # ###### # # # #### +;; +;; Statements + +;;;; Simple Statements + +(print_statement) {} + +(assert_statement) {} + +(expression_statement) {} + +(return_statement (_) @expr) @stmt +{ + edge @stmt.function_returns -> @expr.output +} + +(delete_statement) {} + +(raise_statement) {} + +(pass_statement) {} + +(break_statement) {} + +(continue_statement) {} + +(global_statement) {} + +(nonlocal_statement) {} + +(exec_statement) {} + +(type_alias_statement) {} + +;;;; Compound Statements + +(if_statement) {} + +(if_clause) {} + +(elif_clause) {} + +(else_clause) {} + +(for_statement) {} + +(while_statement) {} + +(try_statement) {} + +(except_group_clause) {} + +(except_clause) {} + +(finally_clause) {} + +(with_statement) {} + +(with_clause) {} + +(function_definition + name: (identifier) @name) { + attr (@name.def) node_definition = @name +} + +(function_definition + name: (identifier) @name + parameters: (parameters) @params + body: (block) @body) @func +{ + node call + node drop_scope + node return_value + + attr (@name.def) definiens_node = @func + edge @func.after_scope -> @name.def + edge @name.def -> call + edge call -> return_value + edge @body.before_scope -> @params.after_scope + edge @body.before_scope -> drop_scope + edge drop_scope -> @func.bottom + attr (drop_scope) type = "drop_scopes" + attr (call) pop_scoped_symbol = "()" + edge @params.before_scope -> JUMP_TO_SCOPE_NODE + attr (return_value) is_exported + let @func.function_returns = return_value + + ; Prevent functions defined inside of method bodies from being treated like methods + let @body.class_self_scope = #null + let @body.class_member_attr_scope = #null +} + +(function_definition + parameters: (parameters + . (identifier) @param) + body: (block) @body) +{ + edge @param.def -> @param.class_self_scope + edge @param.class_member_attr_scope -> @param.output + edge @param.output -> @body.after_scope + attr (@param.output) push_node = @param +} + +(parameters + (_) @param) @params +{ + node @param.param_index + node @param.param_name + + attr (@param.param_index) push_symbol = (named-child-index @param) + edge @param.param_index -> @params.before_scope + edge @params.after_scope -> @param.input + edge @param.param_name -> @params.before_scope +} + + +(parameter/identifier) @param @name +{ + attr (@name.def) node_definition = @name + attr (@param.param_name) push_node = @param + edge @name.def -> @param.param_name + edge @name.def -> @param.param_index + edge @param.input -> @name.def +} + +[ + (parameter/default_parameter + name: (_) @name + value: (_) @value) @param + (parameter/typed_default_parameter + name: (_) @name + value: (_) @value) @param +] { + attr (@name.def) node_definition = @name + attr (@param.param_name) push_node = @name + edge @name.def -> @param.param_name + edge @name.def -> @param.param_index + edge @param.input -> @name.def + edge @name.def -> @value.output +} + +[ + (parameter/typed_parameter + . (_) @name) @param + (parameter/list_splat_pattern + (_) @name) @param + (parameter/dictionary_splat_pattern + (_) @name) @param +] { + attr (@name.def) node_definition = @name + attr (@param.param_name) push_node = @name + edge @name.def -> @param.param_name + edge @name.def -> @param.param_index + edge @param.input -> @name.def +} + +(class_definition) @class { + node @class.call_drop + node @class.member_attrs + node @class.members + node @class.super_scope +} + +(class_definition + name: (identifier) @name) { + attr (@name.def) node_definition = @name +} + +(class_definition + name: (identifier) @name) @class +{ + node call + node self_dot + node self_scope + node members_dot + + attr (@name.def) definiens_node = @class + attr (@name.def) syntax_type = "class" + edge @class.after_scope -> @name.def + edge @name.def -> call + edge @name.def -> members_dot + edge members_dot -> @class.members + edge call -> @class.call_drop + edge @class.call_drop -> self_scope + edge self_scope -> @class.super_scope + edge self_scope -> self_dot + edge self_dot -> @class.members + edge @class.members -> @class.member_attrs + attr (call) pop_scoped_symbol = "()" + attr (@class.call_drop) type = "drop_scopes" + attr (members_dot) pop_symbol = "." + attr (self_dot) pop_symbol = "." + attr (@class.member_attrs) push_symbol = "." + attr (self_scope) is_exported +} + +(class_definition + body: (_) @body) @class +{ + let @body.class_member_attr_scope = @class.member_attrs + node @body.class_parent_scope + edge @body.class_parent_scope -> @class.class_parent_scope + edge @body.class_parent_scope -> @class.local_scope + let @body.class_self_scope = @class.call_drop + let @body.class_super_scope = @class.super_scope +} + +(class_definition + body: (block + (_) @last_stmt .)) @class +{ + edge @class.members -> @last_stmt.after_scope +} + +(class_definition + superclasses: (argument_list + (_) @superclass)) @class +{ + edge @class.super_scope -> @superclass.output +} + +(decorated_definition + definition: (_) @def) @stmt +{ + edge @def.before_scope -> @stmt.before_scope + edge @stmt.after_scope -> @def.after_scope +} + +(match_statement) {} + +(case_clause + (case_pattern) @pattern + consequence: (_) @consequence) +{ + edge @consequence.before_scope -> @pattern.new_bindings +} + +(case_clause + consequence: (_) @consequence) @clause +{ + edge @consequence.before_scope -> @clause.before_scope + edge @clause.after_scope -> @consequence.after_scope +} + +;; +;; ####### +;; # # # ##### ##### ###### #### #### # #### # # #### +;; # # # # # # # # # # # # # ## # # +;; ##### ## # # # # ##### #### #### # # # # # # #### +;; # ## ##### ##### # # # # # # # # # # +;; # # # # # # # # # # # # # # # ## # # +;; ####### # # # # # ###### #### #### # #### # # #### +;; +;; Expressions + +(conditional_expression) {} + +(named_expression) {} + +(as_pattern) {} + +(await) {} + +(binary_operator + (_) @left + (_) @right) @binop +{ + edge @binop.new_bindings -> @left.new_bindings + edge @binop.new_bindings -> @right.new_bindings +} + +(primary_expression/identifier) @name { + attr (@name.ref) node_reference = @name +} + +(primary_expression/identifier) @name +{ + edge @name.output -> @name.local_scope + edge @name.output -> @name.class_parent_scope + edge @name.local_scope -> @name.input + attr (@name.input) pop_node = @name + attr (@name.output) node_reference = @name + + edge @name.new_bindings -> @name.def +} + +(string) {} + +(concatenated_string) {} + +(integer) {} + +(float) {} + +(true) {} + +(false) {} + +(none) {} + +(unary_operator) {} + +(attribute + object: (_) @object + attribute: (identifier) @name) @expr +{ + node input_dot + node output_dot + + edge @expr.output -> @name.output + edge @name.output -> output_dot + edge output_dot -> @object.output + edge @object.input -> input_dot + edge input_dot -> @name.input + edge @name.input -> @expr.input + attr (output_dot) push_symbol = "." + attr (input_dot) pop_symbol = "." + attr (@name.input) pop_node = @name + attr (@name.output) push_node = @name +} + +(primary_expression/attribute + attribute: (identifier) @name) +{ + attr (@name.output) is_reference +} + +(subscript) {} + +(call + function: (_) @fn + arguments: (argument_list) @args) @call +{ + node output_args + + edge @call.output -> output_args + edge output_args -> @fn.output + attr (output_args) push_scoped_symbol = "()", scope = @args.args +} + +(call + function: (attribute + object: (_) @receiver) + arguments: (argument_list + (expression) @arg) @args) +{ + node receiver_arg_index + attr (receiver_arg_index) pop_symbol = "0" + edge @args.args -> receiver_arg_index + edge receiver_arg_index -> @receiver.output + + ;; FIXME the arguments will also exist with their unshifted indices because of the general + ;; rule below! + node arg_index + attr (arg_index) pop_symbol = (plus 1 (named-child-index @arg)) + edge arg_index -> @arg.output +} + +(call + arguments: (argument_list + (keyword_argument + name: (identifier) @name + value: (_) @val)) @args) +{ + node arg_name + edge @args.args -> arg_name + attr (arg_name) pop_node = @name + edge arg_name -> @val.output +} + +(argument_list) @args { + node @args.args + attr (@args.args) is_exported +} + +(argument_list + (expression) @arg) @args +{ + node arg_index + edge @args.args -> arg_index + attr (arg_index) pop_symbol = (named-child-index @arg) + edge arg_index -> @arg.output +} + +( + (call + function: (identifier) @_fn_name) @call + (#eq? @_fn_name "super") +) +{ + edge @call.output -> @call.class_super_scope +} + +(list) @list +{ + node called + + edge @list.output -> called + edge called -> @list.global_dot + attr (called) push_symbol = "list" +} + +(list (_) @el) @list +{ + edge @list.new_bindings -> @el.new_bindings +} + +(list_comprehension) {} + +(dictionary (pair) @pair) @dict +{ + edge @dict.new_bindings -> @pair.new_bindings +} + +(pair + value: (_) @value) @pair +{ + edge @pair.new_bindings -> @value.new_bindings +} + +(dictionary_comprehension) {} + +(set (_) @el) @set +{ + edge @set.new_bindings -> @el.new_bindings +} + +(set_comprehension) {} + +[ + (tuple (_) @element) + (expression_list (_) @element) +] @tuple +{ + node el_index + + edge @tuple.output -> el_index + attr (el_index) pop_symbol = (named-child-index @element) + edge el_index -> @element.output + + edge @tuple.new_bindings -> @element.new_bindings +} + +(parenthesized_expression) {} + +(generator_expression) {} + +(ellipsis) {} + +(list_splat (_) @splatted) @splat +{ + edge @splat.new_bindings -> @splatted.new_bindings +} + +;; +;; ###### +;; # # ## ##### ##### ###### ##### # # #### +;; # # # # # # # # # ## # # +;; ###### # # # # ##### # # # # # #### +;; # ###### # # # ##### # # # # +;; # # # # # # # # # ## # # +;; # # # # # ###### # # # # #### +;; +;; Patterns + + +[ + (pattern/identifier) @name @pattern + ; identifiers in patterns + (as_pattern (identifier) @name .) @pattern + (keyword_pattern (identifier) @name) @pattern + (splat_pattern (identifier) @name) @pattern + ; every context where _simple_pattern is used, because matching + ; pattern/dotted_name does not work + (case_pattern (dotted_name . (_) @name .) @pattern) + (keyword_pattern (dotted_name . (_) @name .) @pattern) + (union_pattern (dotted_name . (_) @name .) @pattern) +] { + attr (@name.def) node_definition = @name + edge @pattern.output -> @pattern.local_scope + edge @pattern.output -> @pattern.class_parent_scope + edge @pattern.local_scope -> @pattern.input + attr (@pattern.local_scope -> @pattern.input) precedence = 1 + attr (@pattern.input) node_definition = @name + attr (@pattern.output) push_node = @name + + edge @pattern.new_bindings -> @name.def +} + +(pattern/subscript) {} + +(pattern/attribute + attribute: (identifier) @name) +{ + attr (@name.input) is_definition +} + + +(list_splat_pattern (_) @pattern) @list { + edge @list.new_bindings -> @pattern.new_bindings +} + +[ + (pattern_list (_) @pattern) + (tuple_pattern (_) @pattern) + (list_pattern (_) @pattern) +] @list +{ + node pattern_index + + let statement_scope = @list.local_scope + node @pattern.local_scope + edge statement_scope -> @pattern.local_scope + attr (statement_scope -> @pattern.local_scope) precedence = (plus 1 (named-child-index @pattern)) + + edge pattern_index -> @list.input + edge @pattern.input -> pattern_index + attr (pattern_index) push_symbol = (named-child-index @pattern) +} + +(class_pattern (case_pattern) @pattern) @class { + edge @class.new_bindings -> @pattern.new_bindings +} + +(splat_pattern (_) @pattern) @splat { + edge @splat.new_bindings -> @pattern.new_bindings +} + +(union_pattern (_) @pattern) @union { + edge @union.new_bindings -> @pattern.new_bindings +} + +(dict_pattern (_) @pattern) @dict { + edge @dict.new_bindings -> @pattern.new_bindings +} + +(complex_pattern) {} + +(as_pattern + (expression) @value + alias: (as_pattern_target (identifier) @name)) @as_pattern +{ + attr (@name.local_scope -> @name.input) precedence = 1 + attr (@name.input) is_definition + + edge @as_pattern.new_bindings -> @value.new_bindings + edge @as_pattern.new_bindings -> @name.new_bindings +} + +(keyword_pattern) {} + +(case_pattern (_) @pattern) @case +{ + edge @case.new_bindings -> @pattern.new_bindings +} + +[ + (assignment + left: (_) @pattern + right: (_) @value) + (with_item + value: + (as_pattern + (_) @value + alias: (as_pattern_target (_) @pattern))) +] +{ + edge @pattern.input -> @value.output +} + +;; +;; ##### ####### +;; # # # # # # ##### ## # # # # # ##### ###### #### +;; # # # ## # # # # # # # # # # # # # +;; ##### # # # # # # # ## # # # # ##### #### +;; # # # # # # ###### ## # # ##### # # +;; # # # # ## # # # # # # # # # # # +;; ##### # # # # # # # # # # # ###### #### +;; +;; Syntax Types + +;; +;; BEGIN BIG GNARLY DISJUNCTION +;; +;; The following pair of rules is intended to capture the following behavior: +;; +;; If a function definition is used to define a method, by being inside a class +;; definition, then we make its syntax type `method`. Otherwise, we make it's +;; syntax type `function`. Unfortunately, because of the limitations on negation +;; and binding in tree sitter queries, we cannot negate `class_definition` or +;; similar things directly. Instead, we have to manually push the negation down +;; to form the finite disjunction it corresponds to. +;; + +[ + (class_definition (block (decorated_definition (function_definition name: (_)@name)))) + (class_definition (block (function_definition name: (_)@name))) +] +{ + attr (@name.def) syntax_type = "method" +} + +[ + (module (decorated_definition (function_definition name: (_)@name))) + (module (function_definition name: (_)@name)) + + (if_statement (block (decorated_definition (function_definition name: (_)@name)))) + (if_statement (block (function_definition name: (_)@name))) + + (elif_clause (block (decorated_definition (function_definition name: (_)@name)))) + (elif_clause (block (function_definition name: (_)@name))) + + (else_clause (block (decorated_definition (function_definition name: (_)@name)))) + (else_clause (block (function_definition name: (_)@name))) + + (case_clause (block (decorated_definition (function_definition name: (_)@name)))) + (case_clause (block (function_definition name: (_)@name))) + + (for_statement (block (decorated_definition (function_definition name: (_)@name)))) + (for_statement (block (function_definition name: (_)@name))) + + (while_statement (block (decorated_definition (function_definition name: (_)@name)))) + (while_statement (block (function_definition name: (_)@name))) + + (try_statement (block (decorated_definition (function_definition name: (_)@name)))) + (try_statement (block (function_definition name: (_)@name))) + + (except_clause (block (decorated_definition (function_definition name: (_)@name)))) + (except_clause (block (function_definition name: (_)@name))) + + (finally_clause (block (decorated_definition (function_definition name: (_)@name)))) + (finally_clause (block (function_definition name: (_)@name))) + + (with_statement (block (decorated_definition (function_definition name: (_)@name)))) + (with_statement (block (function_definition name: (_)@name))) + + (function_definition (block (decorated_definition (function_definition name: (_)@name)))) + (function_definition (block (function_definition name: (_)@name))) +] +{ + attr (@name.def) syntax_type = "function" +} + +;; +;; END BIG GNARLY DISJUNCTION +;; diff --git a/languages/tree-sitter-stack-graphs-python/test/aliased_imports.py b/languages/tree-sitter-stack-graphs-python/test/aliased_imports.py new file mode 100644 index 000000000..0373a33be --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/aliased_imports.py @@ -0,0 +1,31 @@ +#------ path: foo.py ------# + +# module +class A: + a = 1 + +class B: + class C: + class D: + d = 2 + +#------ path: main.py ---# + +from foo import A as X, B.C.D as Y +import foo as f + +print X.a, Y.d +# ^ defined: 5 +# ^ defined: 10 + +print A, B.C +# ^ defined: +# ^ defined: +# ^ defined: + +print f.B +# ^ defined: 3, 15 +# ^ defined: 7 + +print foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/attributes.py b/languages/tree-sitter-stack-graphs-python/test/attributes.py new file mode 100644 index 000000000..6dd3d0e48 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/attributes.py @@ -0,0 +1,14 @@ +a = 1 + +a.b = 2 + +a.c.d = 5 + +print a.b +# ^ defined: 1 +# ^ defined: 3 + +print a.c, a.c.d +# ^ defined: 1 +# ^ defined: +# ^ defined: 5 diff --git a/languages/tree-sitter-stack-graphs-python/test/blocks.py b/languages/tree-sitter-stack-graphs-python/test/blocks.py new file mode 100644 index 000000000..462890baf --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/blocks.py @@ -0,0 +1,21 @@ +def f(): + if a: + b = 1 + else: + c = 2 + + print b, c + # ^ defined: 3 + # ^ defined: 5 + +class G: + if d: + e = 1 + + print e + # ^ defined: 13 + +print b, c, e +# ^ defined: +# ^ defined: +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/chained_functions.py b/languages/tree-sitter-stack-graphs-python/test/chained_functions.py new file mode 100644 index 000000000..ef1dac21d --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/chained_functions.py @@ -0,0 +1,23 @@ +class A: + w = 1 + x = 2 + y = 3 + z = 4 + +def get_a(): + return A +def get_b(): + return get_a() +def get_c(): + return get_b() +def get_d(): + return get_c() +def get_e(): + return get_d() +def get_f(): + return get_e() + +g = get_f(A) +print g.x, g.y +# ^ defined: 3 +# ^ defined: 4 diff --git a/languages/tree-sitter-stack-graphs-python/test/chained_methods.py.skip b/languages/tree-sitter-stack-graphs-python/test/chained_methods.py.skip new file mode 100644 index 000000000..9c4f53115 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/chained_methods.py.skip @@ -0,0 +1,27 @@ +class Builder: + def set_a(self, a): + self.a = a + return self + + def set_b(self, b): + self.b = b + return self + + def set_c(self, c): + self.c = c + return self + + def set_d(self, d): + self.d = d + return self + + def set_e(self, e): + self.d = d + return self + +Builder().set_a('a1').set_b('b2').set_c('c3').set_d('d4').set_e('e4') +# ^ defined: 2 +# ^ defined: 6 +# ^ defined: 10 +# ^ defined: 14 +# ^ defined: 18 diff --git a/languages/tree-sitter-stack-graphs-python/test/class_members.py b/languages/tree-sitter-stack-graphs-python/test/class_members.py new file mode 100644 index 000000000..c67ace992 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/class_members.py @@ -0,0 +1,23 @@ +a = 1 + +class B: + c = a + # ^ defined: 1 + + def d(self): + return self.c + + class E: + f = a + # ^ defined: 1 + +print B.c +# ^ defined: 3 +# ^ defined: 1, 4 + +print B.d(1) +# ^ defined: 7 + +print B.a, E.a +# ^ defined: +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/decorators.py b/languages/tree-sitter-stack-graphs-python/test/decorators.py new file mode 100644 index 000000000..b55f44b2f --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/decorators.py @@ -0,0 +1,10 @@ +from a import deprecated +from b import ignore_warnings + +class A: + @deprecated + # ^ defined: 1 + @ignore_warnings.all + # ^ defined: 2 + def b(self): + pass diff --git a/languages/tree-sitter-stack-graphs-python/test/exceptions.py b/languages/tree-sitter-stack-graphs-python/test/exceptions.py new file mode 100644 index 000000000..b6b2e57e1 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/exceptions.py @@ -0,0 +1,5 @@ +try: + print() +except Exception as e: + x = e + # ^ defined: 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/functions.py b/languages/tree-sitter-stack-graphs-python/test/functions.py new file mode 100644 index 000000000..31a1d6782 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/functions.py @@ -0,0 +1,31 @@ +import a.x.y +import b.x.y + +def get_x(value): + return value.x + # ^ defined: 4 + +print get_x(a).y +# ^ defined: 1 + +print get_x(b).y +# ^ defined: 2 + +def get_a(): + return a + +print get_a(b).x +# ^ defined: 1 + +print get_x(foo=1, value=a).y +# ^ defined: 1 + +def foo(w: int, x, y=1, z: int=4, *args, **dict): + local = x +# ^ defined: 23 + print(args, w, z) +# ^ defined: 23 +# ^ defined: 23 +# ^ defined: 23 + return y +# ^ defined: 23 diff --git a/languages/tree-sitter-stack-graphs-python/test/imported_functions.py b/languages/tree-sitter-stack-graphs-python/test/imported_functions.py new file mode 100644 index 000000000..495697740 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imported_functions.py @@ -0,0 +1,17 @@ +#------ path: a.py ------# + +def foo(x): + return x + +#------ path: b.py ------# + +class A: + bar = 1 + +#------ path: main.py ---------# + +from a import * +from b import * + +foo(A).bar +# ^ defined: 9 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports.py b/languages/tree-sitter-stack-graphs-python/test/imports.py new file mode 100644 index 000000000..96d561813 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports.py @@ -0,0 +1,31 @@ +#------ path: one/two.py -----------# + +# module +import a.b.c + +d = 1 + +e = a.b + +#------ path: three/__init__.py ---# + +# module +f = 3 + +#------ path: main.py -------------# + +from one.two import d, e.c +# ^ defined: 3 +# ^ defined: 6 +# ^ defined: 4, 8 + +import three +# ^ defined: 12 + +print(d, e.c) +# ^ defined: 6, 17 +# ^ defined: 4, 17 + +print three.f +# ^ defined: 12, 22 +# ^ defined: 13 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_submodule.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_submodule.py new file mode 100644 index 000000000..ea51f9450 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_submodule.py @@ -0,0 +1,14 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +from foo import bar + +bar.BAR +# ^ defined: 7, 3 +# ^ defined: 3 + +foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value.py new file mode 100644 index 000000000..1eb97ffe2 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value.py @@ -0,0 +1,13 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +from foo import FOO + +FOO +# ^ defined: 7, 3 + +foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value_aliased.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value_aliased.py new file mode 100644 index 000000000..088ca9544 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value_aliased.py @@ -0,0 +1,13 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +from foo import FOO as QUX + +QUX +# ^ defined: 7, 3 + +FOO +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_wildcard.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_wildcard.py new file mode 100644 index 000000000..3a1de4ffe --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_wildcard.py @@ -0,0 +1,13 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +from foo import * + +FOO +# ^ defined: 3 + +foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_submodule_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_submodule_a_value.py new file mode 100644 index 000000000..8d89d8538 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_submodule_a_value.py @@ -0,0 +1,16 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +from foo.bar import BAR + +BAR +# ^ defined: 7, 3 + +foo +# ^ defined: + +bar +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_a_value.py new file mode 100644 index 000000000..961cd1622 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_a_value.py @@ -0,0 +1,10 @@ +# --- path: foo/__init__.py --- + +FOO = 42 + +# --- path: foo/bar/test.py --- + +from .. import FOO + +FOO +# ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_submodule_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_submodule_a_value.py new file mode 100644 index 000000000..75652f7cd --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_submodule_a_value.py @@ -0,0 +1,10 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: foo/baz/test.py --- + +from ..bar import BAR + +BAR +# ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_a_value.py new file mode 100644 index 000000000..f8de8042c --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_a_value.py @@ -0,0 +1,10 @@ +# --- path: foo/__init__.py --- + +FOO = 42 + +# --- path: foo/test.py --- + +from . import FOO + +FOO +# ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_submodule_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_submodule_a_value.py new file mode 100644 index 000000000..dc64ce387 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_submodule_a_value.py @@ -0,0 +1,10 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: foo/test.py --- + +from .bar import BAR + +BAR +# ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_module.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_module.py new file mode 100644 index 000000000..897acdf63 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_module.py @@ -0,0 +1,11 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +import foo + +foo.FOO +# ^ defined: 7, 3 +# ^ defined: 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_module_aliased.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_module_aliased.py new file mode 100644 index 000000000..220a6b72d --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_module_aliased.py @@ -0,0 +1,14 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +import foo as qux + +qux.FOO +# ^ defined: 7, 3 +# ^ defined: 3 + +foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule.py new file mode 100644 index 000000000..312cd66a1 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule.py @@ -0,0 +1,15 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +import foo.bar + +foo.bar.BAR +# ^ defined: 7 +# ^ defined: 7, 3 +# ^ defined: 3 + +bar +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule_aliased.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule_aliased.py new file mode 100644 index 000000000..b6bfdf7b7 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule_aliased.py @@ -0,0 +1,17 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +import foo.bar as qux + +qux.BAR +# ^ defined: 7, 3 +# ^ defined: 3 + +foo +# ^ defined: + +bar +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/relative_import_resolves_to_itself.py.skip b/languages/tree-sitter-stack-graphs-python/test/imports/relative_import_resolves_to_itself.py.skip new file mode 100644 index 000000000..2cdb2cf66 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/relative_import_resolves_to_itself.py.skip @@ -0,0 +1,14 @@ +# --- path: foo/__init__.py --- +from . import bar +# ^ defined: 6 + +# --- path: foo/bar/__init__.py --- +BAR = 'b' + +# --- path: main.py --- +from foo import bar +# ^ defined: 6 + +bar.BAR +# ^ defined: 9, 6 +# ^ defined: 6 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/require_explicit_submodule_import.py.skip b/languages/tree-sitter-stack-graphs-python/test/imports/require_explicit_submodule_import.py.skip new file mode 100644 index 000000000..a9d12848c --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/require_explicit_submodule_import.py.skip @@ -0,0 +1,18 @@ +# --- path: foo/__init__.py --- + +FOO = 42 + +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +import foo + +foo.FOO +# ^ defined: 11, 3 +# ^ defined: 3 + +foo.bar +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/instance_members.py b/languages/tree-sitter-stack-graphs-python/test/instance_members.py new file mode 100644 index 000000000..38a6b6d75 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/instance_members.py @@ -0,0 +1,38 @@ +#---- path: a.py ------- + +class A: + def __init__(self, b, c): + self.b = b + self.c = A(c, 1) + + def get_b(self): + return self.b + # ^ defined: 4, 5 + + def get_c(self): + return self.c + # ^ defined: 6 + + def get_all(self): + return [self.get_b(), self.get_c()] + # ^ defined: 8 + # ^ defined: 12 + +a = A(1, 2) +a.get_all() +# ^ defined: 16 + +a.b +# ^ defined: 4, 5 + +a.c.b +# ^ defined: 4, 5 +# ^ defined: 6 + +#----- path: main.py --------- + +import a + +print a.A, a.a +# ^ defined: 3 +# ^ defined: 21 diff --git a/languages/tree-sitter-stack-graphs-python/test/loops.py b/languages/tree-sitter-stack-graphs-python/test/loops.py new file mode 100644 index 000000000..2f0663295 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/loops.py @@ -0,0 +1,19 @@ +class Node3: + value = 3 +class Node2: + value = 2 + next = Node3 +class Node1: + value = 1 + next = Node2 + +def linked_list_search(l, item): + node = l + while node: + if node.value == item: + return node + # ^ defined: 10, 11, 16 + node = node.next + +linked_list_search(Node1, 5).value +# ^ defined: 6 diff --git a/languages/tree-sitter-stack-graphs-python/test/many_definitions.py b/languages/tree-sitter-stack-graphs-python/test/many_definitions.py new file mode 100644 index 000000000..f85f3a8ec --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/many_definitions.py @@ -0,0 +1,76 @@ +#--- path: a.py ---# + +def f0(a): return X.a + X.b +def f1(a): return f0(1) +def f2(a): return f1(2) +def f3(a): return f2(3) +def f4(a): return f3(4) +def f5(a): return f4(5) +def f6(a): return f5(6) +def f7(a): return f6(7) +def f8(a): return f7(8) +def f9(a): return f8(9) + +class C1: + def m0(self, b): return f9(0) + def m1(self, b): return self.m0(1) + def m2(self, b): return self.m1(2) + def m3(self, b): return self.m2(3) + def m4(self, b): return self.m3(4) + def m5(self, b): return self.m4(5) + def m6(self, b): return self.m5(6) + def m7(self, b): return self.m6(7) + def m8(self, b): return self.m7(8) + +def f10(): return C1.m8(0) +def f11(): return f10(1) +def f12(): return X.c(2) +def f13(): return X.c(3) +def f14(): return x(4) +def f15(): return x(5) +def f16(): return x(6) +def f17(): return x(7) +def f18(): return x(8) + +class C2: + def m0(self): return X.d(0) + def m1(self): return X.d(1) + def m2(self): return X.d(2) + def m3(self): return X.d(3) + def m4(self): return X.d(4) + def m5(self): return X.d(5) + def m6(self): return X.d(6) + def m7(self): return X.d(7) + def m8(self): return X.d(8) + +#--- path: main.py ---# + +from a import * + +print f0(), f4(), f8() +# ^ defined: 3 +# ^ defined: 7 +# ^ defined: 11 + +print C1.m0, C1().m0(), C1.m4, C1().m4, C1.m8, C1.m8 +# ^ defined: 14 +# ^ defined: 15 +# ^ defined: 15 +# ^ defined: 19 +# ^ defined: 19 +# ^ defined: 23 +# ^ defined: 23 + +print f10(), f14(), f18() +# ^ defined: 25 +# ^ defined: 29 +# ^ defined: 33 + +print C2.m0, C2().m0(), C2.m4, C2().m4, C2.m8, C2.m8 +# ^ defined: 35 +# ^ defined: 36 +# ^ defined: 36 +# ^ defined: 40 +# ^ defined: 40 +# ^ defined: 44 +# ^ defined: 44 diff --git a/languages/tree-sitter-stack-graphs-python/test/pattern_matching.py b/languages/tree-sitter-stack-graphs-python/test/pattern_matching.py new file mode 100644 index 000000000..d6b8980a3 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/pattern_matching.py @@ -0,0 +1,42 @@ +command = 1 +current_room = 2 +character = 3 + +match command.split(): + # ^ defined: 1 + case ["quit"]: + print("Goodbye!") + quit_game() + case ["look"]: + current_room.describe() + # ^ defined: 2 + case ["get", obj]: + character.get(obj, current_room) + # ^ defined: 3 + # ^ defined: 13 + # ^ defined: 2 + case ["go", direction]: + current_room = current_room.neighbor(direction) + # ^ defined: 18 + case { "foo": foo }: + print(foo) + # ^ defined: 21 + case {"bar": bar, "quux": quux}: + print(bar,quux) + # ^ defined: 24 + # ^ defined: 24 + case ["grab", { "key": {"garply": garply}}]: + print(garply) + # ^ defined: 28 + case ["drop", *objs]: + print(objs) + # ^ defined: 31 + case ["get", obj] | ["pick", "up", obj] | ["pick", obj, "up"]: + print(obj) + # ^ defined: 34, 34, 34 + case ["go", ("north" | "south" | "east" | "west") as direction2]: + current_room = current_room.neighbor(direction2) + # ^ defined: 37 + case (foo, "bar"): + print(foo) + # ^ defined: 40 diff --git a/languages/tree-sitter-stack-graphs-python/test/redundant_reexport.py b/languages/tree-sitter-stack-graphs-python/test/redundant_reexport.py new file mode 100644 index 000000000..53afca29e --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/redundant_reexport.py @@ -0,0 +1,15 @@ +#--- path: a/__init__.py ---# + +from . import child + +#--- path: a/child.py ---- + +def f(): + pass + +#--- path: main.py ---# + +import a + +print a.child.f() +# ^ defined: 7 diff --git a/languages/tree-sitter-stack-graphs-python/test/relative_imports.py b/languages/tree-sitter-stack-graphs-python/test/relative_imports.py new file mode 100644 index 000000000..f14571575 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/relative_imports.py @@ -0,0 +1,60 @@ +#------ path: foo/bar/main.py ------# + +from . import a +from .b import B +from ..c import see +from ..c.d import D + +print a.A +# ^ defined: 3, 25 +# ^ defined: 26 + +print B.bee +# ^ defined: 4, 31 +# ^ defined: 32 + +print see() +# ^ defined: 5, 37 + + +print D.d +# ^ defined: 44 + +#------ path: foo/bar/a.py --------# + +# module +A = "a" + +#------ path: foo/bar/b.py --------# + +# module +class B: + bee = 1 + +#------ path: foo/c.py ------------# + +# module +def see(): + pass + +#------ path: foo/c/d.py ---------# + +# module +class D: + d = "d" + +#------ path: foo/e/g.py ---# + +# module +G = 1 + +#------ path: foo/e/__init__.py ---# + +# module +from .g import G +# ^ defined: 49 + +from ..c import see +# ^ defined: 37 + +E = 1 diff --git a/languages/tree-sitter-stack-graphs-python/test/statement_bindings.py b/languages/tree-sitter-stack-graphs-python/test/statement_bindings.py new file mode 100644 index 000000000..6e3a13abb --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/statement_bindings.py @@ -0,0 +1,9 @@ +import a.b +import c.d + +with a as x, c as y: + print x.b, y.d + # ^ defined: 1, 4 + # ^ defined: 1 + # ^ defined: 2, 4 + # ^ defined: 2 diff --git a/languages/tree-sitter-stack-graphs-python/test/superclasses.py b/languages/tree-sitter-stack-graphs-python/test/superclasses.py new file mode 100644 index 000000000..849a192a6 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/superclasses.py @@ -0,0 +1,19 @@ +class A: + def __init__(self): + self.some_attr = 2 + + def some_method(self): + print self + +class B(A): + def method2(self): + print self.some_attr, self.some_method() + # ^ defined: 3 + # ^ defined: 5, 14 + + def some_method(self): + pass + + def other(self): + super().some_method() + # ^ defined: 5 diff --git a/languages/tree-sitter-stack-graphs-python/test/test.py b/languages/tree-sitter-stack-graphs-python/test/test.py new file mode 100644 index 000000000..e69de29bb diff --git a/languages/tree-sitter-stack-graphs-python/test/tuples.py b/languages/tree-sitter-stack-graphs-python/test/tuples.py new file mode 100644 index 000000000..1ea6ee3ef --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/tuples.py @@ -0,0 +1,38 @@ +class a: + x = 1 +class b: + x = 1 +class c: + x = 1 + +(a1, b1) = (a, b) +a2, b2 = (a, b) +(a3, b3) = (a, b) +a4, b4 = a, b + +print a1.x, b1.x +# ^ defined: 2 +# ^ defined: 4 +print a2.x, b2.x +# ^ defined: 2 +# ^ defined: 4 +print a3.x, b3.x +# ^ defined: 2 +# ^ defined: 4 +print a4.x, b4.x +# ^ defined: 2 +# ^ defined: 4 + +t = (a, b), c +(a5, b5), c5 = t + +print a5.x, b5.x, c5.x +# ^ defined: 2 +# ^ defined: 4 +# ^ defined: 6 + +(a6, (b6, c6)) = (a, (b, c)) + +print a6.x, b6.x +# ^ defined: 2 +# ^ defined: 4 diff --git a/languages/tree-sitter-stack-graphs-python/test/wildcard_import.py b/languages/tree-sitter-stack-graphs-python/test/wildcard_import.py new file mode 100644 index 000000000..f2ac040f2 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/wildcard_import.py @@ -0,0 +1,21 @@ +#------ path: one/two.py ------# + +a = 1 +b = 2 + +#------ path: one/three.py ------# + +b = 3 +c = 4 + +#------ path: main.py ---------# + +from one.two import * +from one.three import * + +print a +# ^ defined: 3 +print b +# ^ defined: 8 +print c +# ^ defined: 9 diff --git a/stack-graphs/src/visualization/visualization.js b/stack-graphs/src/visualization/visualization.js index 116850791..6fdab1ef6 100644 --- a/stack-graphs/src/visualization/visualization.js +++ b/stack-graphs/src/visualization/visualization.js @@ -24,6 +24,7 @@ class StackGraph { this.graph = graph; this.paths = paths; + this.cleanup_data(); this.compute_data(); this.current_node = null; @@ -33,6 +34,19 @@ class StackGraph { this.render(); } + cleanup_data() { + let idx = 0; + while (idx < this.graph.edges.length) { + let edge = this.graph.edges[idx]; + if (edge.source.file === edge.sink.file && edge.source.local_id === edge.sink.local_id) { + console.log("ignoring self loop", edge); + this.graph.edges.splice(idx, 1); + } else { + idx += 1; + } + } + } + compute_data() { this.F = {}; this.ID = {}; @@ -47,7 +61,6 @@ class StackGraph { const file = graph.files[i]; this.F[file] = i; } - console.log(this.F); } compute_node_data() { diff --git a/tree-sitter-stack-graphs/Cargo.toml b/tree-sitter-stack-graphs/Cargo.toml index 2de26a86f..d5429304a 100644 --- a/tree-sitter-stack-graphs/Cargo.toml +++ b/tree-sitter-stack-graphs/Cargo.toml @@ -82,4 +82,4 @@ walkdir = { version = "2.3", optional = true } [dev-dependencies] pretty_assertions = "0.7" -tree-sitter-python = "0.19.1" +tree-sitter-python = "=0.19.1"