Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Load prebuilt macros #1856

Merged
merged 3 commits into from
Dec 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,30 @@ jobs:
- name: run tests
run: cargo test -p scarb-metadata

test-prebuilt-plugins:
name: test prebuilt plugins ${{ matrix.platform.name }}
runs-on: ${{ matrix.platform.os }}
# This is isolated, so it can be run on more platforms.
strategy:
fail-fast: false
matrix:
platform:
- name: linux x86-64
os: ubuntu-latest
- name: windows x86-64
os: windows-latest
- name: macos arm64
os: macos-latest
- name: macos x86-64
os: macos-13
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Run prebuilt plugin tests
run: |
cargo test -p scarb --test proc_macro_prebuilt -- --ignored

check-rust:
runs-on: ubuntu-latest
steps:
Expand Down
7 changes: 7 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ xshell = "0.2"
xxhash-rust = { version = "0.8", features = ["xxh3"] }
zip = { version = "0.6", default-features = false, features = ["deflate"] }
zstd = "0.13"
target-triple = "0.1"

# Here we specify real dependency specifications for Cairo crates *if* currently we want to use
# a particular unreleased commit (which is frequent mid-development).
Expand Down
1 change: 1 addition & 0 deletions scarb/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ windows-sys.workspace = true
zstd.workspace = true
cargo_metadata.workspace = true
flate2.workspace = true
target-triple.workspace = true

[target.'cfg(not(target_os = "linux"))'.dependencies]
reqwest = { workspace = true, default-features = true }
Expand Down
18 changes: 11 additions & 7 deletions scarb/src/bin/scarb/commands/proc_macro_server.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use anyhow::Result;
use scarb::ops::CompilationUnitsOpts;
use scarb::{
compiler::{plugin::proc_macro::ProcMacroHost, CairoCompilationUnit, CompilationUnit},
core::{Config, Workspace},
Expand All @@ -15,8 +16,11 @@ pub fn run(config: &mut Config) -> Result<()> {
features: FeaturesSelector::AllFeatures,
no_default_features: false,
},
true,
&ws,
CompilationUnitsOpts {
ignore_cairo_version: true,
load_prebuilt_macros: true,
},
)?;

// Compile procedural macros only.
Expand All @@ -43,12 +47,12 @@ fn load_plugins(
ws: &Workspace<'_>,
proc_macros: &mut ProcMacroHost,
) -> Result<()> {
for plugin_info in unit
.cairo_plugins
.into_iter()
.filter(|plugin_info| !plugin_info.builtin)
{
proc_macros.register(plugin_info.package, ws.config())?;
for plugin_info in unit.cairo_plugins.into_iter().filter(|p| !p.builtin) {
if let Some(prebuilt) = plugin_info.prebuilt {
proc_macros.register_instance(prebuilt);
} else {
proc_macros.register_new(plugin_info.package, ws.config())?;
}
}

Ok(())
Expand Down
13 changes: 10 additions & 3 deletions scarb/src/compiler/compilation_unit.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
use std::fmt::Write;
use std::hash::{Hash, Hasher};

use anyhow::{ensure, Result};
use cairo_lang_filesystem::cfg::CfgSet;
use cairo_lang_filesystem::db::CrateIdentifier;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use smol_str::SmolStr;
use std::fmt::Write;
use std::hash::{Hash, Hasher};
use std::sync::Arc;
use typed_builder::TypedBuilder;

use crate::compiler::plugin::proc_macro::ProcMacroInstance;
use crate::compiler::Profile;
use crate::core::{
ManifestCompilerConfig, Package, PackageId, PackageName, Target, TargetKind, Workspace,
Expand Down Expand Up @@ -72,6 +73,9 @@ pub struct ProcMacroCompilationUnit {

/// Rust compiler configuration parameters to use in this unit.
pub compiler_config: serde_json::Value,

/// Instance of the proc macro loaded from prebuilt library, if available.
pub prebuilt: Option<Arc<ProcMacroInstance>>,
}

/// Information about a single package that is part of a [`CompilationUnit`].
Expand All @@ -96,8 +100,11 @@ pub struct CompilationUnitComponent {
pub struct CompilationUnitCairoPlugin {
/// The Scarb plugin [`Package`] to load.
pub package: Package,
/// Indicate whether the plugin is built into Scarb, or compiled from source.
pub builtin: bool,
pub prebuilt_allowed: bool,
/// Instance of the proc macro loaded from prebuilt library, if available.
pub prebuilt: Option<Arc<ProcMacroInstance>>,
}

/// Unique identifier of the compilation unit component.
Expand Down
13 changes: 7 additions & 6 deletions scarb/src/compiler/db.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
use crate::compiler::plugin::proc_macro::{ProcMacroHost, ProcMacroHostPlugin};
use crate::compiler::{CairoCompilationUnit, CompilationUnitAttributes, CompilationUnitComponent};
use crate::core::Workspace;
use crate::DEFAULT_MODULE_MAIN_FILE;
use anyhow::{anyhow, Result};
use cairo_lang_compiler::db::{RootDatabase, RootDatabaseBuilder};
use cairo_lang_compiler::project::{AllCratesConfig, ProjectConfig, ProjectConfigContent};
Expand All @@ -14,11 +18,6 @@ use std::path::PathBuf;
use std::sync::Arc;
use tracing::trace;

use crate::compiler::plugin::proc_macro::{ProcMacroHost, ProcMacroHostPlugin};
use crate::compiler::{CairoCompilationUnit, CompilationUnitAttributes, CompilationUnitComponent};
use crate::core::Workspace;
use crate::DEFAULT_MODULE_MAIN_FILE;

pub struct ScarbDatabase {
pub db: RootDatabase,
pub proc_macro_host: Arc<ProcMacroHostPlugin>,
Expand Down Expand Up @@ -59,8 +58,10 @@ fn load_plugins(
let plugin = ws.config().cairo_plugins().fetch(package_id)?;
let instance = plugin.instantiate()?;
builder.with_plugin_suite(instance.plugin_suite());
} else if let Some(prebuilt) = &plugin_info.prebuilt {
proc_macros.register_instance(prebuilt.clone());
} else {
proc_macros.register(plugin_info.package.clone(), ws.config())?;
proc_macros.register_new(plugin_info.package.clone(), ws.config())?;
}
}
let macro_host = Arc::new(proc_macros.into_plugin()?);
Expand Down
7 changes: 5 additions & 2 deletions scarb/src/compiler/plugin/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use itertools::Itertools;
use serde::{Deserialize, Serialize};

use crate::compiler::plugin::builtin::BuiltinCairoRunPlugin;
use crate::compiler::plugin::proc_macro::compilation::SharedLibraryProvider;
use crate::core::{Package, PackageId, TargetKind, Workspace};

use self::builtin::{BuiltinStarkNetPlugin, BuiltinTestPlugin};
Expand All @@ -29,8 +30,10 @@ pub fn fetch_cairo_plugin(package: &Package, ws: &Workspace<'_>) -> Result<()> {
assert!(package.is_cairo_plugin());
let target = package.fetch_target(&TargetKind::CAIRO_PLUGIN)?;
let props: CairoPluginProps = target.props()?;
// No need to fetch for buildin plugins.
if !props.builtin {
// There is no need to run `cargo fetch` for builtin plugins.
// The `fetch` will not be run for a proc macro that contains a prebuilt library file.
// Note, that in case the prebuilt lib file is corrupted, it will be later compiled with Cargo anyway.
if !props.builtin && package.prebuilt_lib_path().is_none() {
proc_macro::fetch_crate(package, ws)?;
}
Ok(())
Expand Down
25 changes: 25 additions & 0 deletions scarb/src/compiler/plugin/proc_macro/compilation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,14 @@ use ra_ap_toolchain::Tool;
use scarb_ui::{Message, OutputFormat};
use serde::{Serialize, Serializer};
use serde_json::value::RawValue;
use std::env::consts::DLL_SUFFIX;
use std::fmt::Display;
use std::fs;
use std::io::{Seek, SeekFrom};
use std::ops::Deref;
use std::process::Command;
use tar::Archive;
use target_triple::target;
use tracing::trace_span;

pub const PROC_MACRO_BUILD_PROFILE: &str = "release";
Expand All @@ -31,6 +33,8 @@ pub trait SharedLibraryProvider {
fn target_path(&self, config: &Config) -> Filesystem;
/// Location of the shared library for the package.
fn shared_lib_path(&self, config: &Config) -> Result<Utf8PathBuf>;
/// Location of the prebuilt binary for the package, if defined.
fn prebuilt_lib_path(&self) -> Option<Utf8PathBuf>;
}

impl SharedLibraryProvider for Package {
Expand Down Expand Up @@ -61,6 +65,27 @@ impl SharedLibraryProvider for Package {
.path_unchecked()
.join(lib_name))
}

fn prebuilt_lib_path(&self) -> Option<Utf8PathBuf> {
let target_triple = target!();

let prebuilt_name = format!(
"{name}_v{version}_{target}{suffix}",
name = self.id.name,
version = self.id.version,
target = target_triple,
suffix = DLL_SUFFIX
);

let prebuilt_path = self
.root()
.join("target")
.join("scarb")
.join("cairo-plugin")
.join(prebuilt_name);

prebuilt_path.exists().then_some(prebuilt_path)
}
}

pub fn compile_unit(unit: ProcMacroCompilationUnit, ws: &Workspace<'_>) -> Result<()> {
Expand Down
24 changes: 18 additions & 6 deletions scarb/src/compiler/plugin/proc_macro/ffi.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use crate::core::{Config, Package, PackageId};
use crate::core::{Package, PackageId};
use anyhow::{ensure, Context, Result};
use cairo_lang_defs::patcher::PatchBuilder;
use cairo_lang_macro::{
Expand Down Expand Up @@ -26,6 +26,7 @@ use libloading::os::unix::Symbol as RawSymbol;
#[cfg(windows)]
use libloading::os::windows::Symbol as RawSymbol;
use smol_str::SmolStr;
use tracing::trace;

pub trait FromSyntaxNode {
fn from_syntax_node(db: &dyn SyntaxGroup, node: &impl TypedSyntaxNode) -> Self;
Expand Down Expand Up @@ -61,11 +62,22 @@ impl Debug for ProcMacroInstance {

impl ProcMacroInstance {
/// Load shared library
pub fn try_new(package: Package, config: &Config) -> Result<Self> {
let lib_path = package
.shared_lib_path(config)
.context("could not resolve shared library path")?;
let plugin = unsafe { Plugin::try_new(lib_path.to_path_buf())? };
pub fn try_new(package_id: PackageId, lib_path: Utf8PathBuf) -> Result<Self> {
trace!("loading compiled macro for `{}` package", package_id);
let plugin = unsafe { Plugin::try_new(lib_path)? };
Ok(Self {
expansions: unsafe { Self::load_expansions(&plugin, package_id)? },
package_id,
plugin,
})
}

pub fn try_load_prebuilt(package: Package) -> Result<Self> {
trace!("loading prebuilt macro for `{}` package", package.id);
let prebuilt_path = package
.prebuilt_lib_path()
.context("could not resolve prebuilt library path")?;
let plugin = unsafe { Plugin::try_new(prebuilt_path)? };
Ok(Self {
expansions: unsafe { Self::load_expansions(&plugin, package.id)? },
package_id: package.id,
Expand Down
16 changes: 12 additions & 4 deletions scarb/src/compiler/plugin/proc_macro/host.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
use crate::compiler::plugin::proc_macro::compilation::SharedLibraryProvider;
use crate::compiler::plugin::proc_macro::{
Expansion, ExpansionKind, FromSyntaxNode, ProcMacroInstance,
};
use crate::core::{Config, Package, PackageId};
use anyhow::{ensure, Result};
use anyhow::{ensure, Context, Result};
use cairo_lang_defs::ids::{ModuleItemId, TopLevelLanguageElementId};
use cairo_lang_defs::patcher::{PatchBuilder, RewriteNode};
use cairo_lang_defs::plugin::{
Expand Down Expand Up @@ -1150,9 +1151,16 @@ pub struct ProcMacroHost {
}

impl ProcMacroHost {
pub fn register(&mut self, package: Package, config: &Config) -> Result<()> {
let instance = ProcMacroInstance::try_new(package, config)?;
self.macros.push(Arc::new(instance));
pub fn register_instance(&mut self, instance: Arc<ProcMacroInstance>) {
self.macros.push(instance);
}

pub fn register_new(&mut self, package: Package, config: &Config) -> Result<()> {
let lib_path = package
.shared_lib_path(config)
.context("could not resolve shared library path")?;
let instance = ProcMacroInstance::try_new(package.id, lib_path)?;
self.register_instance(Arc::new(instance));
Ok(())
}

Expand Down
Loading
Loading