From 2d832e1ebb53728f55c71c474b7daf1cdb1cd7d8 Mon Sep 17 00:00:00 2001 From: Justus Adam Date: Mon, 22 Jul 2024 18:26:43 -0400 Subject: [PATCH] Enable all cargo tests in CI (#151) ## What Changed and Why? - Renames the `cargo make` rule run in CI to `ci-tests`. Under the hood this now runs a general `cargo test` so that all tests are run. The only way `cargo` tests are now excluded from CI is via `ignore` instead of having a second filter via the CI script. This ensures all tests are actually exercised in CI. - Converts the `consts` module into local structs `Markers` and `Symbols` that are initialized per-session. This fixes an issue with stale data that can cause test cases to fail. This is caused by `Symbol`s. A `Symbol` is an interned string, with the interner stored in a thread-local `SESSION_GLOBALS` variable. The "costants" we allocate in `consts` however are `lazy_static`, which means they are shared across threads. If two compiler sessions run in parallel, then one would initialize the values in `consts`, interning `Symbol`s in its interner. However the indices in those symbols would refer to different strings in the other session, which breaks the annotation extractor. A similar scenario would also occur if the two sessions are sequential, as the `lazy_static` would preserve the symbols. This is why the fresh, per-session intialization is needed, and simply changing `lazy_static` to `thread_local` would be insufficient. - Switches the name generation for test crates in the integration tests to using random names. This fixes issues where the crates already existed or tests wold race for the same name if the test code was the same. - It disables incremental compilation in CI. This substantially reduces the disk space needed and stays within the limits of disk space available in [GitHub hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories). ## Checklist - [x] Above description has been filled out so that upon quash merge we have a good record of what changed. - [x] New functions, methods, types are documented. Old documentation is updated if necessary - [x] Documentation in Notion has been updated - [x] Tests for new behaviors are provided - [ ] New test suites (if any) ave been added to the CI tests (in `.github/workflows/rust.yml`) either as compiler test or integration test. *Or* justification for their omission from CI has been provided in this PR description. --- .github/workflows/rust.yml | 149 +++++++++--------- Cargo.lock | 45 +++++- Cargo.toml | 4 +- Makefile.toml | 42 ++--- crates/flowistry_pdg/build.rs | 23 +++ crates/flowistry_pdg_construction/build.rs | 23 +++ .../flowistry_pdg_construction/src/graph.rs | 4 +- crates/paralegal-flow/build.rs | 9 +- crates/paralegal-flow/src/ann/db.rs | 90 ++++++++--- crates/paralegal-flow/src/ann/parse.rs | 45 +++++- crates/paralegal-flow/src/consts.rs | 36 ----- crates/paralegal-flow/src/discover.rs | 10 +- crates/paralegal-flow/src/lib.rs | 1 - crates/paralegal-flow/src/test_utils.rs | 4 +- crates/paralegal-policy/Cargo.toml | 1 + crates/paralegal-policy/build.rs | 23 +++ crates/paralegal-policy/tests/helpers/mod.rs | 39 ++--- 17 files changed, 339 insertions(+), 209 deletions(-) create mode 100644 crates/flowistry_pdg/build.rs create mode 100644 crates/flowistry_pdg_construction/build.rs delete mode 100644 crates/paralegal-flow/src/consts.rs create mode 100644 crates/paralegal-policy/build.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6e6331bafc..aff462a842 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -2,13 +2,17 @@ name: Rust on: push: - branches: [ "main" ] + branches: ["main"] pull_request: - branches: [ "main" ] + branches: ["main"] env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + # Disable incremental computation. Has no benefit in CI, as the builds are + # always fresh. Substantially reduces disk usage (17GB -> 6GB as of writing + # this) + CARGO_INCREMENTAL: false jobs: compiler-tests: @@ -17,93 +21,86 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - # just triggers rustup to download the toolchain - - name: Cache Toolchain - uses: actions/cache@v3 - with: - path: ~/.rustup - key: ${{ runner.os }}-rust-toolchain-${{ hashFiles('rust-toolchain.toml') }} - - name: Cache Dependencies - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-rust-deps-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }} - - name: Install cargo-make - run: rustup run stable cargo install --force --debug cargo-make - - name: Build - run: cargo make install - - name: Run tests - run: cargo make pdg-tests + - uses: actions/checkout@v3 + - name: Cache Toolchain + uses: actions/cache@v3 + with: + path: ~/.rustup + key: ${{ runner.os }}-rust-toolchain-${{ hashFiles('rust-toolchain.toml') }} + - name: Cache Dependencies + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-rust-deps-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }} + - name: Install cargo-make + run: rustup run stable cargo install --force --debug cargo-make + - name: Run tests + run: cargo make ci-tests format-check: name: Format Control runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - # just triggers rustup to download the toolchain - - name: Cache Toolchain - uses: actions/cache@v3 - with: - path: ~/.rustup - key: ${{ runner.os }}-rust-toolchain-${{ hashFiles('rust-toolchain.toml') }} - - name: Install cargo-make - run: rustup run stable cargo install --force --debug cargo-make - - name: Run checks - run: cargo make format-check-all + - uses: actions/checkout@v3 + - name: Cache Toolchain + uses: actions/cache@v3 + with: + path: ~/.rustup + key: ${{ runner.os }}-rust-toolchain-${{ hashFiles('rust-toolchain.toml') }} + - name: Install cargo-make + run: rustup run stable cargo install --force --debug cargo-make + - name: Run checks + run: cargo make format-check-all linting: name: Clippy runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Cache Toolchain - uses: actions/cache@v3 - with: - path: ~/.rustup - key: ${{ runner.os }}-rust-toolchain-${{ hashFiles('rust-toolchain.toml') }} - - name: Cache Dependencies - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-rust-deps-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }} - - name: Install cargo-make - run: rustup run stable cargo install --force --debug cargo-make - - name: Here come the complaints - run: cargo make clippy-check-all + - uses: actions/checkout@v3 + - name: Cache Toolchain + uses: actions/cache@v3 + with: + path: ~/.rustup + key: ${{ runner.os }}-rust-toolchain-${{ hashFiles('rust-toolchain.toml') }} + - name: Cache Dependencies + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-rust-deps-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }} + - name: Install cargo-make + run: rustup run stable cargo install --force --debug cargo-make + - name: Here come the complaints + run: cargo make clippy-check-all documentation: name: Documentation Test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Cache Toolchain - uses: actions/cache@v3 - with: - path: ~/.rustup - key: ${{ runner.os }}-rust-toolchain-${{ hashFiles('rust-toolchain.toml') }} - - name: Cache Dependencies - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-rust-deps-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }} - - name: Install cargo-make - run: rustup run stable cargo install --force --debug cargo-make - - name: Here come the complaints - run: cargo make doc-check \ No newline at end of file + - uses: actions/checkout@v3 + - name: Cache Toolchain + uses: actions/cache@v3 + with: + path: ~/.rustup + key: ${{ runner.os }}-rust-toolchain-${{ hashFiles('rust-toolchain.toml') }} + - name: Cache Dependencies + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-rust-deps-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }} + - name: Install cargo-make + run: rustup run stable cargo install --force --debug cargo-make + - name: Here come the complaints + run: cargo make doc-check diff --git a/Cargo.lock b/Cargo.lock index 5cdb2eea30..7a906b405b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -949,6 +949,7 @@ dependencies = [ "paralegal-flow", "paralegal-spdg", "petgraph", + "rand", "serde_json", "simple_logger", "strum", @@ -1027,6 +1028,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "pretty" version = "0.11.3" @@ -1063,6 +1070,36 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -1112,12 +1149,12 @@ name = "rustc_plugin" version = "0.7.4-nightly-2023-08-25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1348edfa020dbe4807a4d99272332dadcbbedff6b587accb95faefe20d2c7129" -replace = "rustc_plugin 0.7.4-nightly-2023-08-25 (git+https://github.com/JustusAdam/rustc_plugin?rev=6cc378d254e0c4fdfdbec975f82302173bfa0586)" +replace = "rustc_plugin 0.7.4-nightly-2023-08-25 (git+https://github.com/JustusAdam/rustc_plugin?rev=6dcee5d8758f8ac456209e95dfa25e03142295f1)" [[package]] name = "rustc_plugin" version = "0.7.4-nightly-2023-08-25" -source = "git+https://github.com/JustusAdam/rustc_plugin?rev=6cc378d254e0c4fdfdbec975f82302173bfa0586#6cc378d254e0c4fdfdbec975f82302173bfa0586" +source = "git+https://github.com/JustusAdam/rustc_plugin?rev=6dcee5d8758f8ac456209e95dfa25e03142295f1#6dcee5d8758f8ac456209e95dfa25e03142295f1" dependencies = [ "cargo_metadata", "log", @@ -1138,12 +1175,12 @@ name = "rustc_utils" version = "0.7.4-nightly-2023-08-25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09428c7086894369685cca54a516acc0f0ab6d0e5a628c094ba83bfddaf1aedf" -replace = "rustc_utils 0.7.4-nightly-2023-08-25 (git+https://github.com/JustusAdam/rustc_plugin?rev=6cc378d254e0c4fdfdbec975f82302173bfa0586)" +replace = "rustc_utils 0.7.4-nightly-2023-08-25 (git+https://github.com/JustusAdam/rustc_plugin?rev=6dcee5d8758f8ac456209e95dfa25e03142295f1)" [[package]] name = "rustc_utils" version = "0.7.4-nightly-2023-08-25" -source = "git+https://github.com/JustusAdam/rustc_plugin?rev=6cc378d254e0c4fdfdbec975f82302173bfa0586#6cc378d254e0c4fdfdbec975f82302173bfa0586" +source = "git+https://github.com/JustusAdam/rustc_plugin?rev=6dcee5d8758f8ac456209e95dfa25e03142295f1#6dcee5d8758f8ac456209e95dfa25e03142295f1" dependencies = [ "anyhow", "cfg-if", diff --git a/Cargo.toml b/Cargo.toml index e29e9c6aab..92f78d90c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,9 +29,9 @@ debug = true [replace."rustc_utils:0.7.4-nightly-2023-08-25"] # path = "../rustc_plugin/crates/rustc_utils" git = "https://github.com/JustusAdam/rustc_plugin" -rev = "6cc378d254e0c4fdfdbec975f82302173bfa0586" +rev = "6dcee5d8758f8ac456209e95dfa25e03142295f1" [replace."rustc_plugin:0.7.4-nightly-2023-08-25"] # path = "../rustc_plugin/crates/rustc_plugin" git = "https://github.com/JustusAdam/rustc_plugin" -rev = "6cc378d254e0c4fdfdbec975f82302173bfa0586" +rev = "6dcee5d8758f8ac456209e95dfa25e03142295f1" diff --git a/Makefile.toml b/Makefile.toml index 7d201ae6e9..fb7d26cf4c 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -22,9 +22,9 @@ default_to_workspace = false command = "cargo" args = ["install", "--locked", "--path", "crates/paralegal-flow"] -[tasks.pdg-tests] -description = "The suite of synthetic tests for the PDG and the policy framweork." -dependencies = ["analyzer-tests", "policy-framework-tests", "guide-project"] +[tasks.ci-tests] +description = "The suite of tests we run in CI" +dependencies = ["cargo-tests", "guide-project"] [tasks.check-all] description = "Perform all formatting-like checks. This is the same set that runs in the CI." @@ -47,38 +47,28 @@ stage them and rerun the script until no more errors occur. """ dependencies = ["format-all", "clippy-all"] -[tasks.analyzer-tests] -args = [ - "test", - "-p", - "paralegal-flow", - "--test", - "non_transitive_graph_tests", - "--test", - "call_chain_analysis_tests", - "--test", - "control_flow_tests", - "--test", - "new_alias_analysis_tests", - "--test", - "boxes", - "--test", - "async_tests", - "--no-fail-fast", -] -description = "Low-level tests for the PDG emitted by the analyzer specifically." +[tasks.cargo-tests] +command = "cargo" +args = ["test"] + +[tasks.flowistry-tests] +command = "cargo" +args = ["test", "-pflowistry_pdg", "-pflowistry_pdg_construction"] + +[tasks.paralegal-flow-tests] command = "cargo" +args = ["test", "-pparalegal-flow", "-pparalegal-spdg"] -[tasks.policy-framework-tests] -description = "Tests related to the correctness of the policy framework." +[tasks.policy-tests] command = "cargo" -args = ["test", "-p", "paralegal-policy", "--lib"] +args = ["test", "-pparalegal-policy"] [tasks.guide-project] description = "Build and run the policy from the guide." cwd = "guide/deletion-policy" command = "cargo" args = ["run"] +dependencies = ["install"] [tasks.doc-check] dependencies = [ diff --git a/crates/flowistry_pdg/build.rs b/crates/flowistry_pdg/build.rs new file mode 100644 index 0000000000..0c1063561b --- /dev/null +++ b/crates/flowistry_pdg/build.rs @@ -0,0 +1,23 @@ +use std::env; +use std::path::PathBuf; + +fn rustup_toolchain_path() -> PathBuf { + let rustup_home = env::var("RUSTUP_HOME").unwrap(); + let rustup_tc = env::var("RUSTUP_TOOLCHAIN").unwrap(); + [&rustup_home, "toolchains", &rustup_tc] + .into_iter() + .collect() +} + +fn get_rustup_lib_path() -> PathBuf { + let mut rustup_lib = rustup_toolchain_path(); + rustup_lib.push("lib"); + rustup_lib +} + +fn main() { + if cfg!(target_os = "linux") { + let rustup_lib = get_rustup_lib_path(); + println!("cargo:rustc-link-search=native={}", rustup_lib.display()); + } +} diff --git a/crates/flowistry_pdg_construction/build.rs b/crates/flowistry_pdg_construction/build.rs new file mode 100644 index 0000000000..0c1063561b --- /dev/null +++ b/crates/flowistry_pdg_construction/build.rs @@ -0,0 +1,23 @@ +use std::env; +use std::path::PathBuf; + +fn rustup_toolchain_path() -> PathBuf { + let rustup_home = env::var("RUSTUP_HOME").unwrap(); + let rustup_tc = env::var("RUSTUP_TOOLCHAIN").unwrap(); + [&rustup_home, "toolchains", &rustup_tc] + .into_iter() + .collect() +} + +fn get_rustup_lib_path() -> PathBuf { + let mut rustup_lib = rustup_toolchain_path(); + rustup_lib.push("lib"); + rustup_lib +} + +fn main() { + if cfg!(target_os = "linux") { + let rustup_lib = get_rustup_lib_path(); + println!("cargo:rustc-link-search=native={}", rustup_lib.display()); + } +} diff --git a/crates/flowistry_pdg_construction/src/graph.rs b/crates/flowistry_pdg_construction/src/graph.rs index 19b13121b1..7fc27431ce 100644 --- a/crates/flowistry_pdg_construction/src/graph.rs +++ b/crates/flowistry_pdg_construction/src/graph.rs @@ -223,8 +223,8 @@ impl<'tcx> DepGraph<'tcx> { dot::Dot::with_attr_getters( &self.graph, &[], - &|_, _| format!("fontname=\"Courier New\""), - &|_, (_, _)| format!("fontname=\"Courier New\",shape=box") + &|_, _| "fontname=\"Courier New\"".to_string(), + &|_, (_, _)| "fontname=\"Courier New\",shape=box".to_string(), ) ); rustc_utils::mir::body::run_dot(path.as_ref(), graph_dot.into_bytes()) diff --git a/crates/paralegal-flow/build.rs b/crates/paralegal-flow/build.rs index 7fb9ee8ce6..b0775e2148 100644 --- a/crates/paralegal-flow/build.rs +++ b/crates/paralegal-flow/build.rs @@ -25,13 +25,18 @@ fn rustup_toolchain_path() -> PathBuf { .collect() } +fn get_rustup_lib_path() -> PathBuf { + let mut rustup_lib = rustup_toolchain_path(); + rustup_lib.push("lib"); + rustup_lib +} + /// Taken from Kani /// () /// this code links the rustc libraries directly with the compiled binaries. pub fn link_rustc_lib() { // Add rustup to the rpath in order to properly link with the correct rustc version. - let mut rustup_lib = rustup_toolchain_path(); - rustup_lib.push("lib"); + let rustup_lib = get_rustup_lib_path(); add_link_search_path_for_compiler_binaries(rustup_lib.display()); // While we hard-code the above for development purposes, for a release/install we look diff --git a/crates/paralegal-flow/src/ann/db.rs b/crates/paralegal-flow/src/ann/db.rs index d529781f59..ddf10f6dbe 100644 --- a/crates/paralegal-flow/src/ann/db.rs +++ b/crates/paralegal-flow/src/ann/db.rs @@ -13,9 +13,10 @@ use crate::{ ann::{Annotation, MarkerAnnotation}, args::{Args, MarkerControl}, - consts, + discover::AttrMatchT, + sym_vec, utils::{ - resolve::expect_resolve_string_to_def_id, AsFnAndArgs, InstanceExt, IntoDefId, IntoHirId, + resolve::expect_resolve_string_to_def_id, AsFnAndArgs, InstanceExt, IntoDefId, MetaItemMatch, TyCtxtExt, TyExt, }, Either, HashMap, HashSet, @@ -25,6 +26,7 @@ use flowistry_pdg_construction::{ utils::{try_monomorphize, try_resolve_function}, }; use paralegal_spdg::Identifier; +use rustc_span::Symbol; use rustc_utils::cache::Cache; use rustc_ast::Attribute; @@ -37,8 +39,40 @@ use rustc_middle::{ use std::rc::Rc; +use super::parse::Symbols; + type ExternalMarkers = HashMap>; +/// These are pseudo-constants that are used to match annotations. In theory +/// they never change but they use [`Symbol`] inside, which is only valid so +/// long as the rustc `SESSION_GLOBALS` are live so we need to calculate them +/// afresh in `new`. +struct Markers { + /// This will match the annotation `#[paralegal_flow::label(...)]` when using + /// [`MetaItemMatch::match_extract`](crate::utils::MetaItemMatch::match_extract) + label_marker: AttrMatchT, + /// This will match the annotation `#[paralegal_flow::marker(...)]` when using + /// [`MetaItemMatch::match_extract`](crate::utils::MetaItemMatch::match_extract) + marker_marker: AttrMatchT, + /// This will match the annotation `#[paralegal_flow::output_types(...)]` when using + /// [`MetaItemMatch::match_extract`](crate::utils::MetaItemMatch::match_extract) + otype_marker: AttrMatchT, + /// This will match the annotation `#[paralegal_flow::exception(...)]` when using + /// [`MetaItemMatch::match_extract`](crate::utils::MetaItemMatch::match_extract) + exception_marker: AttrMatchT, +} + +impl Default for Markers { + fn default() -> Self { + Markers { + label_marker: sym_vec!["paralegal_flow", "label"], + marker_marker: sym_vec!["paralegal_flow", "marker"], + otype_marker: sym_vec!["paralegal_flow", "output_types"], + exception_marker: sym_vec!["paralegal_flow", "exception"], + } + } +} + /// The marker context is a database which can be queried as to whether /// functions or types carry markers, whether markers are reachable in bodies, /// etc. @@ -466,6 +500,8 @@ pub struct MarkerDatabase<'tcx> { /// Configuration options _config: &'static MarkerControl, type_markers: Cache, Box>, + markers: Markers, + symbols: Symbols, } impl<'tcx> MarkerDatabase<'tcx> { @@ -478,16 +514,16 @@ impl<'tcx> MarkerDatabase<'tcx> { reachable_markers: Default::default(), _config: args.marker_control(), type_markers: Default::default(), + markers: Markers::default(), + symbols: Default::default(), } } /// Retrieve and parse the local annotations for this item. pub fn retrieve_local_annotations_for(&mut self, def_id: LocalDefId) { let tcx = self.tcx; - let hir = tcx.hir(); - let id = def_id.force_into_hir_id(tcx); - for a in hir.attrs(id) { - match try_parse_annotation(tcx, a) { + for a in tcx.get_attrs_unchecked(def_id.to_def_id()) { + match self.try_parse_annotation(a) { Ok(anns) => { let mut anns = anns.peekable(); if anns.peek().is_some() { @@ -503,27 +539,29 @@ impl<'tcx> MarkerDatabase<'tcx> { } } } -} -fn try_parse_annotation( - tcx: TyCtxt, - a: &Attribute, -) -> Result, String> { - use crate::ann::parse::{ann_match_fn, match_exception, otype_ann_match}; - let one = |a| Either::Left(Some(a)); - let ann = if let Some(i) = a.match_get_ref(&consts::MARKER_MARKER) { - one(Annotation::Marker(ann_match_fn(i)?)) - } else if let Some(i) = a.match_get_ref(&consts::LABEL_MARKER) { - warn!("The `paralegal_flow::label` annotation is deprecated, use `paralegal_flow::marker` instead"); - one(Annotation::Marker(ann_match_fn(i)?)) - } else if let Some(i) = a.match_get_ref(&consts::OTYPE_MARKER) { - Either::Right(otype_ann_match(i, tcx)?.into_iter().map(Annotation::OType)) - } else if let Some(i) = a.match_get_ref(&consts::EXCEPTION_MARKER) { - one(Annotation::Exception(match_exception(i)?)) - } else { - Either::Left(None) - }; - Ok(ann.into_iter()) + fn try_parse_annotation( + &self, + a: &Attribute, + ) -> Result, String> { + let consts = &self.markers; + let tcx = self.tcx; + use crate::ann::parse::{ann_match_fn, match_exception, otype_ann_match}; + let one = |a| Either::Left(Some(a)); + let ann = if let Some(i) = a.match_get_ref(&consts.marker_marker) { + one(Annotation::Marker(ann_match_fn(&self.symbols, i)?)) + } else if let Some(i) = a.match_get_ref(&consts.label_marker) { + warn!("The `paralegal_flow::label` annotation is deprecated, use `paralegal_flow::marker` instead"); + one(Annotation::Marker(ann_match_fn(&self.symbols, i)?)) + } else if let Some(i) = a.match_get_ref(&consts.otype_marker) { + Either::Right(otype_ann_match(i, tcx)?.into_iter().map(Annotation::OType)) + } else if let Some(i) = a.match_get_ref(&consts.exception_marker) { + one(Annotation::Exception(match_exception(&self.symbols, i)?)) + } else { + Either::Left(None) + }; + Ok(ann.into_iter()) + } } type RawExternalMarkers = HashMap>; diff --git a/crates/paralegal-flow/src/ann/parse.rs b/crates/paralegal-flow/src/ann/parse.rs index 96e022c066..d267b8829a 100644 --- a/crates/paralegal-flow/src/ann/parse.rs +++ b/crates/paralegal-flow/src/ann/parse.rs @@ -12,7 +12,7 @@ use super::{ ExceptionAnnotation, MarkerAnnotation, MarkerRefinement, MarkerRefinementKind, VerificationHash, }; use crate::{ - consts, utils, + utils, utils::{write_sep, Print, TinyBitSet}, Symbol, }; @@ -28,6 +28,28 @@ pub extern crate nom; use nom::{error::Error, Parser}; +pub struct Symbols { + /// The symbol `arguments` which we use for refinement in a `#[paralegal_flow::marker(...)]` + /// annotation. + arg_sym: Symbol, + /// The symbol `return` which we use for refinement in a `#[paralegal_flow::marker(...)]` + /// annotation. + return_sym: Symbol, + /// The symbol `verification_hash` which we use for refinement in a + /// `#[paralegal_flow::exception(...)]` annotation. + verification_hash_sym: Symbol, +} + +impl Default for Symbols { + fn default() -> Self { + Self { + arg_sym: Symbol::intern("arguments"), + return_sym: Symbol::intern("return"), + verification_hash_sym: Symbol::intern("verification_hash"), + } + } +} + /// Just a newtype-wrapper for `CursorRef` so we can implement traits on it /// (specifically [`nom::InputLength`]). /// @@ -248,14 +270,17 @@ pub(crate) fn otype_ann_match(ann: &ast::AttrArgs, tcx: TyCtxt) -> Result Result { +pub(crate) fn match_exception( + symbols: &Symbols, + ann: &rustc_ast::AttrArgs, +) -> Result { use rustc_ast::*; match ann { ast::AttrArgs::Delimited(dargs) => { let p = |i| { let (i, verification_hash) = nom::combinator::opt(nom::sequence::preceded( nom::sequence::tuple(( - assert_identifier(*consts::VERIFICATION_HASH_SYM), + assert_identifier(symbols.verification_hash_sym), assert_token(TokenKind::Eq), )), lit(token::LitKind::Str, |s| { @@ -278,21 +303,21 @@ pub(crate) fn match_exception(ann: &rustc_ast::AttrArgs) -> Result R { +fn refinements_parser<'a>(symbols: &Symbols, i: I<'a>) -> R<'a, MarkerRefinement> { nom::combinator::map_res( // nom::multi::separated_list0( // assert_token(TokenKind::Comma), nom::branch::alt(( nom::sequence::preceded( nom::sequence::tuple(( - assert_identifier(*consts::ARG_SYM), + assert_identifier(symbols.arg_sym), assert_token(TokenKind::Eq), )), nom::combinator::map(tiny_bitset, MarkerRefinementKind::Argument), ), nom::combinator::value( MarkerRefinementKind::Return, - assert_identifier(*consts::RETURN_SYM), + assert_identifier(symbols.return_sym), ), )), //), @@ -305,7 +330,10 @@ fn refinements_parser(i: I) -> R { } /// Parser for a [`LabelAnnotation`] -pub(crate) fn ann_match_fn(ann: &rustc_ast::AttrArgs) -> Result { +pub(crate) fn ann_match_fn( + symbols: &Symbols, + ann: &rustc_ast::AttrArgs, +) -> Result { use rustc_ast::*; use token::*; match ann { @@ -313,7 +341,8 @@ pub(crate) fn ann_match_fn(ann: &rustc_ast::AttrArgs) -> Result { stats: Stats, pub marker_ctx: MarkerDatabase<'tcx>, + /// This will match the annotation `#[paralegal_flow::analyze]` when using + /// [`MetaItemMatch::match_extract`](crate::utils::MetaItemMatch::match_extract) + analyze_marker: AttrMatchT, } /// A function we will be targeting to analyze with @@ -77,6 +82,7 @@ impl<'tcx> CollectingVisitor<'tcx> { functions_to_analyze, marker_ctx: MarkerDatabase::init(tcx, opts), stats, + analyze_marker: sym_vec!["paralegal_flow", "analyze"], } } @@ -102,7 +108,7 @@ impl<'tcx> CollectingVisitor<'tcx> { .hir() .attrs(self.tcx.local_def_id_to_hir_id(ident)) .iter() - .any(|a| a.matches_path(&consts::ANALYZE_MARKER)) + .any(|a| a.matches_path(&self.analyze_marker)) } } diff --git a/crates/paralegal-flow/src/lib.rs b/crates/paralegal-flow/src/lib.rs index 54b3d2fab3..fe6fae559e 100644 --- a/crates/paralegal-flow/src/lib.rs +++ b/crates/paralegal-flow/src/lib.rs @@ -65,7 +65,6 @@ mod stats; //mod sah; #[macro_use] pub mod utils; -pub mod consts; #[cfg(feature = "test")] pub mod test_utils; diff --git a/crates/paralegal-flow/src/test_utils.rs b/crates/paralegal-flow/src/test_utils.rs index 8ea2b21fd1..837df0f590 100644 --- a/crates/paralegal-flow/src/test_utils.rs +++ b/crates/paralegal-flow/src/test_utils.rs @@ -246,7 +246,7 @@ impl InlineTestBuilder { let pdg = memo.run(tcx).unwrap(); let graph = PreFrg::from_description(pdg); let cref = graph.ctrl(&self.ctrl_name); - check(cref) + check(cref); }); } } @@ -345,7 +345,7 @@ impl PreFrg { use_rustc(|| { let desc = ProgramDescription::canonical_read(format!( "{dir}/{}", - crate::consts::FLOW_GRAPH_OUT_NAME + paralegal_spdg::FLOW_GRAPH_OUT_NAME )) .unwrap(); Self::from_description(desc) diff --git a/crates/paralegal-policy/Cargo.toml b/crates/paralegal-policy/Cargo.toml index 7f4522aa93..7092e27843 100644 --- a/crates/paralegal-policy/Cargo.toml +++ b/crates/paralegal-policy/Cargo.toml @@ -22,3 +22,4 @@ indexmap = "2.2.6" [dev-dependencies] paralegal-flow = { path = "../paralegal-flow", features = ["test"] } +rand = "0.8.5" diff --git a/crates/paralegal-policy/build.rs b/crates/paralegal-policy/build.rs new file mode 100644 index 0000000000..0c1063561b --- /dev/null +++ b/crates/paralegal-policy/build.rs @@ -0,0 +1,23 @@ +use std::env; +use std::path::PathBuf; + +fn rustup_toolchain_path() -> PathBuf { + let rustup_home = env::var("RUSTUP_HOME").unwrap(); + let rustup_tc = env::var("RUSTUP_TOOLCHAIN").unwrap(); + [&rustup_home, "toolchains", &rustup_tc] + .into_iter() + .collect() +} + +fn get_rustup_lib_path() -> PathBuf { + let mut rustup_lib = rustup_toolchain_path(); + rustup_lib.push("lib"); + rustup_lib +} + +fn main() { + if cfg!(target_os = "linux") { + let rustup_lib = get_rustup_lib_path(); + println!("cargo:rustc-link-search=native={}", rustup_lib.display()); + } +} diff --git a/crates/paralegal-policy/tests/helpers/mod.rs b/crates/paralegal-policy/tests/helpers/mod.rs index 00dad610c3..54d6763a54 100644 --- a/crates/paralegal-policy/tests/helpers/mod.rs +++ b/crates/paralegal-policy/tests/helpers/mod.rs @@ -26,25 +26,29 @@ lazy_static::lazy_static! { build_cmd.current_dir(flow_dir); let stat = build_cmd.status().unwrap(); assert!(stat.success()); - let tool_path = dir.parent().unwrap().parent().unwrap().join("target").join("release").join("cargo-paralegal-flow"); + let tool_path = dir + .parent() + .unwrap() + .parent() + .unwrap() + .join("target") + .join("release") + .join("cargo-paralegal-flow"); assert!(tool_path.exists(), "{}", tool_path.display()); tool_path }; } -fn temporary_directory(to_hash: &impl Hash) -> Result { +fn temporary_directory() -> Result { let tmpdir = env::temp_dir(); - let mut hasher = DefaultHasher::new(); - to_hash.hash(&mut hasher); - let t = SystemTime::now().duration_since(UNIX_EPOCH)?; - t.hash(&mut hasher); - let hash = hasher.finish(); - let short_hash = hash % 0x1_000_000; - let path = tmpdir.join(format!("test-crate-{short_hash:06x}")); - if !path.exists() { - fs::create_dir(&path)?; + loop { + let name: u32 = rand::random(); + let path = tmpdir.join(format!("test-crate-{name:x}")); + if !path.exists() { + fs::create_dir(&path)?; + return Ok(path); + } } - Ok(path) } #[must_use] @@ -71,16 +75,7 @@ fn ensure_run_success(cmd: &mut Command) -> Result<()> { impl Test { pub fn new(code: impl Into) -> Result { let code = code.into(); - let tempdir = temporary_directory(&code)?; - for entry in fs::read_dir(&tempdir)? { - let f = entry?; - let typ = f.file_type()?; - if typ.is_dir() { - fs::remove_dir_all(f.path())?; - } else if typ.is_file() { - fs::remove_file(f.path())?; - } - } + let tempdir = temporary_directory()?; println!("Running in {}", tempdir.display()); Ok(Self { code,