From 488791c3fb08f35923136c72a3fbd6e1fc7b2cee Mon Sep 17 00:00:00 2001 From: Abhishek Shah Date: Mon, 1 Apr 2024 16:26:11 +0530 Subject: [PATCH 1/5] pop_db: init -- use cache on pop build --- Cargo.lock | 115 +++++++++++++++++++++++++++----- Cargo.toml | 1 + src/commands/build/parachain.rs | 7 +- src/commands/new/parachain.rs | 3 + src/db.rs | 86 ++++++++++++++++++++++++ src/engines/parachain_engine.rs | 2 +- src/main.rs | 6 +- 7 files changed, 196 insertions(+), 24 deletions(-) create mode 100644 src/db.rs diff --git a/Cargo.lock b/Cargo.lock index 512a3f78..5441cd36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1575,6 +1575,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -1600,7 +1609,7 @@ dependencies = [ "crossterm_winapi", "libc", "mio", - "parking_lot", + "parking_lot 0.12.1", "signal-hook", "signal-hook-mio", "winapi", @@ -1851,7 +1860,7 @@ dependencies = [ "hashbrown 0.14.3", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.9", ] [[package]] @@ -1954,7 +1963,7 @@ dependencies = [ "futures", "libc", "log", - "parking_lot", + "parking_lot 0.12.1", "pin-project", "serde", "serde_json", @@ -2684,7 +2693,7 @@ checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "windows-sys 0.52.0", ] @@ -2794,6 +2803,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fslock" version = "0.1.8" @@ -2934,6 +2953,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.12.4" @@ -4094,7 +4122,7 @@ dependencies = [ "json-patch", "k8s-openapi", "kube-client", - "parking_lot", + "parking_lot 0.12.1", "pin-project", "serde", "serde_json", @@ -4204,7 +4232,7 @@ dependencies = [ "multihash", "multistream-select", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "pin-project", "quick-protobuf", "rand 0.8.5", @@ -4262,7 +4290,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ "bitflags 2.4.2", "libc", - "redox_syscall", + "redox_syscall 0.4.1", ] [[package]] @@ -5012,6 +5040,17 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -5019,7 +5058,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -5030,7 +5083,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "smallvec", "windows-targets 0.48.5", ] @@ -5434,6 +5487,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sled", "sp-core 30.0.0", "sp-weights", "strum 0.26.1", @@ -5745,6 +5799,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -6887,6 +6950,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot 0.11.2", +] + [[package]] name = "smallvec" version = "1.13.1" @@ -7077,7 +7156,7 @@ dependencies = [ "log", "lru", "no-std-net", - "parking_lot", + "parking_lot 0.12.1", "pin-project", "rand 0.8.5", "rand_chacha 0.3.1", @@ -7113,7 +7192,7 @@ dependencies = [ "log", "lru", "no-std-net", - "parking_lot", + "parking_lot 0.12.1", "pin-project", "rand 0.8.5", "rand_chacha 0.3.1", @@ -7244,7 +7323,7 @@ dependencies = [ "log", "merlin 2.0.1", "parity-scale-codec", - "parking_lot", + "parking_lot 0.12.1", "paste", "primitive-types", "rand 0.8.5", @@ -7291,7 +7370,7 @@ dependencies = [ "log", "merlin 3.0.0", "parity-scale-codec", - "parking_lot", + "parking_lot 0.12.1", "paste", "primitive-types", "rand 0.8.5", @@ -7449,7 +7528,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd4bf9e5fa486416c92c2bb497b7ce2c43eac80cbdc407ffe2d34b365694ac29" dependencies = [ "parity-scale-codec", - "parking_lot", + "parking_lot 0.12.1", "sp-core 30.0.0", "sp-externalities 0.27.0", ] @@ -7565,7 +7644,7 @@ dependencies = [ "hash-db", "log", "parity-scale-codec", - "parking_lot", + "parking_lot 0.12.1", "rand 0.8.5", "smallvec", "sp-core 30.0.0", @@ -7656,7 +7735,7 @@ dependencies = [ "memory-db", "nohash-hasher", "parity-scale-codec", - "parking_lot", + "parking_lot 0.12.1", "rand 0.8.5", "scale-info", "schnellru", @@ -8716,7 +8795,7 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2 0.5.6", @@ -9127,7 +9206,7 @@ dependencies = [ "ipconfig", "lazy_static", "lru-cache", - "parking_lot", + "parking_lot 0.12.1", "resolv-conf", "serde", "smallvec", diff --git a/Cargo.toml b/Cargo.toml index 211269c4..05d38fa0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ duct = "0.13" git2 = "0.18" log = "0.4" # semver = "1.0.20" +sled = "0.34.7" strum = "0.26" strum_macros = "0.26" tempfile = "3.8" diff --git a/src/commands/build/parachain.rs b/src/commands/build/parachain.rs index 20485f56..f26bae42 100644 --- a/src/commands/build/parachain.rs +++ b/src/commands/build/parachain.rs @@ -1,4 +1,4 @@ -use crate::style::{style, Theme}; +use crate::{db::PopDb, style::{style, Theme}}; use clap::Args; use cliclack::{clear_screen, intro, outro, set_theme}; use std::path::PathBuf; @@ -16,11 +16,12 @@ pub struct BuildParachainCommand { } impl BuildParachainCommand { - pub(crate) fn execute(&self) -> anyhow::Result<()> { + pub(crate) fn execute(self) -> anyhow::Result<()> { clear_screen()?; intro(format!("{}: Building a parachain", style(" Pop CLI ").black().on_magenta()))?; set_theme(Theme); - build_parachain(&self.path)?; + let cached_path = PopDb::open_or_init().get_parachain_path(); + build_parachain(self.path.or(cached_path))?; outro("Build Completed Successfully!")?; Ok(()) diff --git a/src/commands/new/parachain.rs b/src/commands/new/parachain.rs index c4e49e4d..2b12a886 100644 --- a/src/commands/new/parachain.rs +++ b/src/commands/new/parachain.rs @@ -2,6 +2,7 @@ use crate::{ engines::parachain_engine::{instantiate_template_dir, Config}, helpers::git_init, style::{style, Theme}, + db::PopDb, }; use clap::{Args, Parser}; use std::{fs, path::Path}; @@ -84,6 +85,8 @@ impl NewParachainCommand { } } spinner.stop("Generation complete"); + let pop_db = PopDb::open_or_init(); + pop_db.set_parachain_path(destination_path); outro(format!("cd into \"{}\" and enjoy hacking! 🚀", &self.name))?; Ok(()) } diff --git a/src/db.rs b/src/db.rs new file mode 100644 index 00000000..a04f0318 --- /dev/null +++ b/src/db.rs @@ -0,0 +1,86 @@ +//! An embedded database meant to be used as a temporary cache to look up +//! and store pop cli args such as commonly used file paths. +//! +//! DB functions should be as error-free as possible. +//! The caching is meant to improve the pop experience and should not impact usage +//! All errors are non-fatal and meant to be discarded unless logged at debug or error log target. +//! +//! This database is implemented using the [sled] crate, which provides an +//! in-memory LSM tree. The keys are paths as `String`s, and the values are +//! the absolute path to the file as a `String`. + +use log::{debug, error}; +use sled::{Config, Db}; +use std::path::{Path, PathBuf}; + +const DB_PATH: &'static str = "/tmp/pop-cli-db"; + +/// The idea is to provide a default path to use based on the previous invocation of `pop new parachain` +/// to make following commands more convinient to use by allowing the user to omit path arguments +/// such as `-p ` or `-r `. This should not effect existing functionality meaning +/// For any pop subcommand, the first priority is the user specified path argument. In the absence of that +/// the second natural place to look is the current working directory and identify it as a parachain. +/// If the current working directory is not detected as a parachain, only then PopDb maybe consulted +/// +/// Usage: +/// First open (or initialize) it : `PopDb::open_or_init()`. +/// On `pop new parachain` cache the dir path by calling `pop_db.set_parachain_path(path)` +/// On `pop new contract` cache the dir path by calling `pop_db.set_contract_path(path)` +/// +/// Use these paths for subcommands like `pop build | test | add | new ` +/// when invoked outside of the parachain directory +pub struct PopDb { + /// May be None due to an error in `open_or_init` + inner: Option, +} +impl PopDb { + /// Try to open a database or create one if it doesn't exist: `Some(db)`. + /// If errors occur, run pop-cli without a database: `None`. + pub(crate) fn open_or_init() -> Self { + let db_path = Path::new(DB_PATH); + if !db_path.exists() { + debug!("{} does not exist, creating database", db_path.display()); + let _ = std::fs::create_dir_all(db_path).map_err(|err| { + error!( + "Failed to create database directory {}\nDue to : {}", + db_path.display(), + err + ) + }); + } + Self { + inner: Config::new() + .path(DB_PATH) + // Set cache capacity to 10 MB + .cache_capacity(10 * 1024 * 1024) + .open() + .map_err(|err| error!("Failed to open database\nDue to : {}", err)) + .ok(), + } + } + + /// Set parachain path + pub(crate) fn set_parachain_path(&self, path: &Path) { + if let Some(ref db) = self.inner { + let path = match path.canonicalize() { + Ok(p) => p, + Err(err) => { + error!("Failed to canonicalize {}\nDue to : {}", path.display(), err); + return; + }, + }; + let _ = db + .insert(b"parachain", path.to_string_lossy().as_bytes()) + .map_err(|err| error!("Failed to set key parachain in database\nDue to : {}", err)); + } + } + /// Get most recent parachain path + pub(crate) fn get_parachain_path(&self) -> Option { + if let Some(ref db) = self.inner { + if let Some(Some(parachain_path)) = db.get(b"parachain").ok() { + return Some(PathBuf::from(String::from_utf8_lossy(¶chain_path).to_string())); + } + } + None + } +} diff --git a/src/engines/parachain_engine.rs b/src/engines/parachain_engine.rs index cf55776d..a33511b3 100644 --- a/src/engines/parachain_engine.rs +++ b/src/engines/parachain_engine.rs @@ -69,7 +69,7 @@ pub fn instantiate_base_template(target: &Path, config: Config) -> Result<()> { Ok(()) } -pub fn build_parachain(path: &Option) -> anyhow::Result<()> { +pub fn build_parachain(path: Option) -> anyhow::Result<()> { cmd("cargo", vec!["build", "--release"]) .dir(path.clone().unwrap_or("./".into())) .run()?; diff --git a/src/main.rs b/src/main.rs index a6ba9e95..fe42e4a5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,6 +15,8 @@ mod parachains; #[cfg(feature = "contract")] mod signer; +mod db; + use anyhow::{anyhow, Result}; use clap::{Parser, Subcommand}; use std::{fs::create_dir_all, path::PathBuf}; @@ -51,7 +53,7 @@ enum Commands { #[tokio::main] async fn main() -> Result<()> { let cli = Cli::parse(); - match &cli.command { + match cli.command { Commands::New(args) => match &args.command { #[cfg(feature = "parachain")] commands::new::NewCommands::Parachain(cmd) => cmd.execute(), @@ -60,7 +62,7 @@ async fn main() -> Result<()> { #[cfg(feature = "contract")] commands::new::NewCommands::Contract(cmd) => cmd.execute(), }, - Commands::Build(args) => match &args.command { + Commands::Build(args) => match args.command { #[cfg(feature = "parachain")] commands::build::BuildCommands::Parachain(cmd) => cmd.execute(), #[cfg(feature = "contract")] From b108e76fe2bc2d39d3d8e2639f6c9cfaa10d7b80 Mon Sep 17 00:00:00 2001 From: Abhishek Shah Date: Mon, 8 Apr 2024 13:01:55 +0530 Subject: [PATCH 2/5] Add todo: for parachain projection detection issue #91 --- src/commands/build/contract.rs | 1 - src/commands/build/parachain.rs | 6 ++---- src/engines/parachain_engine.rs | 13 +++++++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/commands/build/contract.rs b/src/commands/build/contract.rs index 34e7e6fc..50b19ebd 100644 --- a/src/commands/build/contract.rs +++ b/src/commands/build/contract.rs @@ -17,7 +17,6 @@ impl BuildContractCommand { clear_screen()?; intro(format!("{}: Building a contract", style(" Pop CLI ").black().on_magenta()))?; set_theme(Theme); - build_smart_contract(&self.path)?; outro("Build completed successfully!")?; Ok(()) diff --git a/src/commands/build/parachain.rs b/src/commands/build/parachain.rs index f26bae42..8dc637bd 100644 --- a/src/commands/build/parachain.rs +++ b/src/commands/build/parachain.rs @@ -1,4 +1,4 @@ -use crate::{db::PopDb, style::{style, Theme}}; +use crate::style::{style, Theme}; use clap::Args; use cliclack::{clear_screen, intro, outro, set_theme}; use std::path::PathBuf; @@ -20,9 +20,7 @@ impl BuildParachainCommand { clear_screen()?; intro(format!("{}: Building a parachain", style(" Pop CLI ").black().on_magenta()))?; set_theme(Theme); - let cached_path = PopDb::open_or_init().get_parachain_path(); - build_parachain(self.path.or(cached_path))?; - + build_parachain(self.path)?; outro("Build Completed Successfully!")?; Ok(()) } diff --git a/src/engines/parachain_engine.rs b/src/engines/parachain_engine.rs index 3be2476c..84cc2a37 100644 --- a/src/engines/parachain_engine.rs +++ b/src/engines/parachain_engine.rs @@ -74,10 +74,15 @@ pub fn instantiate_base_template(target: &Path, config: Config) -> Result