From 81cf22dab528fdd3ba1d9362aeb5924aa2989aac Mon Sep 17 00:00:00 2001 From: Eric Swanson <64809312+ericswanson-dfinity@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:02:35 -0600 Subject: [PATCH] feat: extension install uses dependencies.json (#3816) --- CHANGELOG.md | 2 + src/dfx-core/src/error/extension.rs | 32 +++++--- src/dfx-core/src/extension/manager/install.rs | 31 +++++--- .../manifest/compatibility_matrix.rs | 77 ------------------ .../src/extension/manifest/dependencies.rs | 78 ++++++++++++++++++- src/dfx-core/src/extension/manifest/mod.rs | 5 -- src/dfx-core/src/extension/mod.rs | 2 + src/dfx-core/src/extension/url.rs | 20 +++++ src/dfx/src/commands/extension/install.rs | 8 +- 9 files changed, 147 insertions(+), 108 deletions(-) delete mode 100644 src/dfx-core/src/extension/manifest/compatibility_matrix.rs create mode 100644 src/dfx-core/src/extension/url.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 12ab34c8d6..5c1bb43037 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ The [NIST guidelines](https://pages.nist.gov/800-63-3/sp800-63b.html) require pa This is now enforced when creating new identities. Identities protected by a shorter password can still be decrypted. +### feat: `dfx extension install` now uses the extension's dependencies.json file to pick the highest compatible version + # 0.21.0 ### feat: dfx killall diff --git a/src/dfx-core/src/error/extension.rs b/src/dfx-core/src/error/extension.rs index 17e4e75f0c..97f89ed5cd 100644 --- a/src/dfx-core/src/error/extension.rs +++ b/src/dfx-core/src/error/extension.rs @@ -105,7 +105,7 @@ pub enum InstallExtensionError { GetExtensionArchiveName(#[from] GetExtensionArchiveNameError), #[error(transparent)] - FindLatestExtensionCompatibleVersion(#[from] FindLatestExtensionCompatibleVersionError), + GetHighestCompatibleVersion(#[from] GetHighestCompatibleVersionError), #[error(transparent)] GetExtensionDownloadUrl(#[from] GetExtensionDownloadUrlError), @@ -124,23 +124,33 @@ pub enum GetExtensionArchiveNameError { } #[derive(Error, Debug)] -pub enum FindLatestExtensionCompatibleVersionError { - #[error("DFX version '{0}' is not supported.")] - DfxVersionNotFoundInCompatibilityJson(semver::Version), +pub enum GetHighestCompatibleVersionError { + #[error(transparent)] + GetDependencies(#[from] GetDependenciesError), - #[error("Extension '{0}' (version '{1}') not found for DFX version {2}.")] - ExtensionVersionNotFoundInRepository(String, semver::Version, String), + #[error("No compatible version found.")] + NoCompatibleVersionFound(), - #[error("Cannot parse compatibility.json due to malformed semver '{0}'")] - MalformedVersionsEntryForExtensionInCompatibilityMatrix(String, #[source] semver::Error), + #[error(transparent)] + DfxOnlyPossibleDependency(#[from] DfxOnlySupportedDependency), +} - #[error("Cannot find compatible extension for dfx version '{1}': compatibility.json (downloaded from '{0}') has empty list of extension versions.")] - ListOfVersionsForExtensionIsEmpty(String, semver::Version), +#[derive(Error, Debug)] +pub enum GetDependenciesError { + #[error(transparent)] + ParseUrl(#[from] url::ParseError), #[error(transparent)] - FetchExtensionCompatibilityMatrix(#[from] FetchExtensionCompatibilityMatrixError), + Get(WrappedReqwestError), + + #[error(transparent)] + ParseJson(WrappedReqwestError), } +#[derive(Error, Debug)] +#[error("'dfx' is the only supported dependency")] +pub struct DfxOnlySupportedDependency; + #[derive(Error, Debug)] #[error("Failed to parse extension manifest URL '{url}'")] pub struct GetExtensionDownloadUrlError { diff --git a/src/dfx-core/src/extension/manager/install.rs b/src/dfx-core/src/extension/manager/install.rs index a48833ba35..b2762fa474 100644 --- a/src/dfx-core/src/extension/manager/install.rs +++ b/src/dfx-core/src/extension/manager/install.rs @@ -1,10 +1,12 @@ use crate::error::extension::{ DownloadAndInstallExtensionToTempdirError, FinalizeInstallationError, - FindLatestExtensionCompatibleVersionError, GetExtensionArchiveNameError, - GetExtensionDownloadUrlError, InstallExtensionError, + GetExtensionArchiveNameError, GetExtensionDownloadUrlError, GetHighestCompatibleVersionError, + InstallExtensionError, }; use crate::error::reqwest::WrappedReqwestError; -use crate::extension::{manager::ExtensionManager, manifest::ExtensionCompatibilityMatrix}; +use crate::extension::{ + manager::ExtensionManager, manifest::ExtensionDependencies, url::ExtensionJsonUrl, +}; use crate::http::get::get_with_retries; use backoff::exponential::ExponentialBackoff; use flate2::read::GzDecoder; @@ -24,9 +26,10 @@ impl ExtensionManager { pub async fn install_extension( &self, extension_name: &str, + url: &ExtensionJsonUrl, install_as: Option<&str>, version: Option<&Version>, - ) -> Result<(), InstallExtensionError> { + ) -> Result { let effective_extension_name = install_as.unwrap_or(extension_name); if self @@ -40,13 +43,15 @@ impl ExtensionManager { let extension_version = match version { Some(version) => version.clone(), - None => self.get_highest_compatible_version(extension_name).await?, + None => self.get_highest_compatible_version(url).await?, }; let github_release_tag = get_git_release_tag(extension_name, &extension_version); let extension_archive = get_extension_archive_name(extension_name)?; - let url = get_extension_download_url(&github_release_tag, &extension_archive)?; + let archive_url = get_extension_download_url(&github_release_tag, &extension_archive)?; - let temp_dir = self.download_and_unpack_extension_to_tempdir(url).await?; + let temp_dir = self + .download_and_unpack_extension_to_tempdir(archive_url) + .await?; self.finalize_installation( extension_name, @@ -55,7 +60,7 @@ impl ExtensionManager { temp_dir, )?; - Ok(()) + Ok(extension_version) } /// Removing the prerelease tag and build metadata, because they should @@ -70,11 +75,13 @@ impl ExtensionManager { async fn get_highest_compatible_version( &self, - extension_name: &str, - ) -> Result { - let manifest = ExtensionCompatibilityMatrix::fetch().await?; + url: &ExtensionJsonUrl, + ) -> Result { + let dependencies = ExtensionDependencies::fetch(url).await?; let dfx_version = self.dfx_version_strip_semver(); - manifest.find_latest_compatible_extension_version(extension_name, &dfx_version) + dependencies + .find_highest_compatible_version(&dfx_version)? + .ok_or(GetHighestCompatibleVersionError::NoCompatibleVersionFound()) } async fn download_and_unpack_extension_to_tempdir( diff --git a/src/dfx-core/src/extension/manifest/compatibility_matrix.rs b/src/dfx-core/src/extension/manifest/compatibility_matrix.rs deleted file mode 100644 index c71bd5cccc..0000000000 --- a/src/dfx-core/src/extension/manifest/compatibility_matrix.rs +++ /dev/null @@ -1,77 +0,0 @@ -use crate::error::extension::{ - FetchExtensionCompatibilityMatrixError, - FetchExtensionCompatibilityMatrixError::{ - CompatibilityMatrixFetchError, MalformedCompatibilityMatrix, - }, - FindLatestExtensionCompatibleVersionError, - FindLatestExtensionCompatibleVersionError::{ - DfxVersionNotFoundInCompatibilityJson, ExtensionVersionNotFoundInRepository, - ListOfVersionsForExtensionIsEmpty, MalformedVersionsEntryForExtensionInCompatibilityMatrix, - }, -}; -use schemars::JsonSchema; -use semver::Version; -use serde::Deserialize; -use std::collections::HashMap; - -pub static COMMON_EXTENSIONS_MANIFEST_LOCATION: &str = - "https://raw.githubusercontent.com/dfinity/dfx-extensions/main/compatibility.json"; - -type DfxVersion = Version; -type ExtensionName = String; - -#[derive(Deserialize, JsonSchema, Debug)] -pub struct ExtensionCompatibilityMatrix( - pub HashMap>, -); - -#[derive(Deserialize, JsonSchema, Debug, Clone)] -pub struct ExtensionCompatibleVersions { - pub versions: Vec, -} - -impl ExtensionCompatibilityMatrix { - pub async fn fetch() -> Result { - let resp = reqwest::get(COMMON_EXTENSIONS_MANIFEST_LOCATION) - .await - .map_err(|e| { - CompatibilityMatrixFetchError(COMMON_EXTENSIONS_MANIFEST_LOCATION.to_string(), e) - })?; - - resp.json().await.map_err(MalformedCompatibilityMatrix) - } - - pub fn find_latest_compatible_extension_version( - &self, - extension_name: &str, - dfx_version: &Version, - ) -> Result { - let manifests = self - .0 - .get(dfx_version) - .ok_or_else(|| DfxVersionNotFoundInCompatibilityJson(dfx_version.clone()))?; - - let extension_location = manifests.get(extension_name).ok_or_else(|| { - ExtensionVersionNotFoundInRepository( - extension_name.to_string(), - dfx_version.clone(), - COMMON_EXTENSIONS_MANIFEST_LOCATION.to_string(), - ) - })?; - let mut extension_versions = vec![]; - for ext_verion in extension_location.versions.iter().rev() { - let version = Version::parse(ext_verion).map_err(|e| { - MalformedVersionsEntryForExtensionInCompatibilityMatrix(ext_verion.to_string(), e) - })?; - extension_versions.push(version); - } - extension_versions.sort(); - extension_versions.reverse(); - extension_versions.first().cloned().ok_or_else(|| { - ListOfVersionsForExtensionIsEmpty( - COMMON_EXTENSIONS_MANIFEST_LOCATION.to_string(), - dfx_version.clone(), - ) - }) - } -} diff --git a/src/dfx-core/src/extension/manifest/dependencies.rs b/src/dfx-core/src/extension/manifest/dependencies.rs index 07755097a0..65d37ea5a2 100644 --- a/src/dfx-core/src/extension/manifest/dependencies.rs +++ b/src/dfx-core/src/extension/manifest/dependencies.rs @@ -1,8 +1,14 @@ +use crate::error::extension::{DfxOnlySupportedDependency, GetDependenciesError}; +use crate::error::reqwest::WrappedReqwestError; +use crate::extension::url::ExtensionJsonUrl; +use crate::http::get::get_with_retries; use crate::json::structure::VersionReqWithJsonSchema; +use backoff::exponential::ExponentialBackoff; use candid::Deserialize; use schemars::JsonSchema; use semver::Version; use std::collections::HashMap; +use std::time::Duration; type ExtensionVersion = Version; type DependencyName = String; @@ -19,6 +25,51 @@ pub struct ExtensionDependencies( pub HashMap>, ); +impl ExtensionDependencies { + pub async fn fetch(url: &ExtensionJsonUrl) -> Result { + let dependencies_json_url = url.to_dependencies_json()?; + let retry_policy = ExponentialBackoff { + max_elapsed_time: Some(Duration::from_secs(60)), + ..Default::default() + }; + let resp = get_with_retries(dependencies_json_url, retry_policy) + .await + .map_err(GetDependenciesError::Get)?; + + resp.json() + .await + .map_err(|e| GetDependenciesError::ParseJson(WrappedReqwestError(e))) + } + + pub fn find_highest_compatible_version( + &self, + dfx_version: &Version, + ) -> Result, DfxOnlySupportedDependency> { + let mut keys: Vec<&Version> = self.0.keys().collect(); + keys.sort(); + keys.reverse(); // check higher extension versions first + + for key in keys { + let dependencies = self.0.get(key).unwrap(); + for (dependency, requirements) in dependencies { + if dependency == "dfx" { + match requirements { + DependencyRequirement::Version(req) => { + if req.matches(dfx_version) { + return Ok(Some(key.clone())); + } + } + } + } else { + return Err(DfxOnlySupportedDependency); + } + } + } + + Ok(None) + } +} + #[test] fn parse_test_file() { let f = r#" @@ -32,6 +83,11 @@ fn parse_test_file() { "dfx": { "version": ">=0.9.6" } + }, + "0.7.0": { + "dfx": { + "version": ">=0.9.9" + } } } "#; @@ -40,9 +96,10 @@ fn parse_test_file() { let manifest = m.unwrap(); let versions = manifest.0.keys().collect::>(); - assert_eq!(versions.len(), 2); + assert_eq!(versions.len(), 3); assert!(versions.contains(&&Version::new(0, 3, 4))); assert!(versions.contains(&&Version::new(0, 6, 2))); + assert!(versions.contains(&&Version::new(0, 7, 0))); let v_3_4 = manifest.0.get(&Version::new(0, 3, 4)).unwrap(); let dfx = v_3_4.get("dfx").unwrap(); @@ -55,4 +112,23 @@ fn parse_test_file() { let DependencyRequirement::Version(req) = dfx; assert!(req.matches(&semver::Version::new(0, 9, 6))); assert!(!req.matches(&semver::Version::new(0, 9, 5))); + + assert_eq!( + manifest + .find_highest_compatible_version(&Version::new(0, 8, 5)) + .unwrap(), + Some(Version::new(0, 3, 4)) + ); + assert_eq!( + manifest + .find_highest_compatible_version(&Version::new(0, 9, 6)) + .unwrap(), + Some(Version::new(0, 6, 2)) + ); + assert_eq!( + manifest + .find_highest_compatible_version(&Version::new(0, 9, 10)) + .unwrap(), + Some(Version::new(0, 7, 0)) + ); } diff --git a/src/dfx-core/src/extension/manifest/mod.rs b/src/dfx-core/src/extension/manifest/mod.rs index 690f69313e..7eb28145d7 100644 --- a/src/dfx-core/src/extension/manifest/mod.rs +++ b/src/dfx-core/src/extension/manifest/mod.rs @@ -1,13 +1,8 @@ //! Directory contains code that parses the .json files. -pub mod compatibility_matrix; pub mod dependencies; pub mod extension; -/// `compatibility.json` is a file describing the compatibility -/// matrix between extensions versions and the dfx version. -pub use compatibility_matrix::ExtensionCompatibilityMatrix; - /// A file that lists the dependencies of all versions of an extension. pub use dependencies::ExtensionDependencies; diff --git a/src/dfx-core/src/extension/mod.rs b/src/dfx-core/src/extension/mod.rs index 2b283274c7..71f15ec03f 100644 --- a/src/dfx-core/src/extension/mod.rs +++ b/src/dfx-core/src/extension/mod.rs @@ -1,5 +1,7 @@ pub mod manager; pub mod manifest; +pub mod url; + use crate::error::extension::ConvertExtensionIntoClapCommandError; use crate::extension::{manager::ExtensionManager, manifest::ExtensionManifest}; use clap::Command; diff --git a/src/dfx-core/src/extension/url.rs b/src/dfx-core/src/extension/url.rs new file mode 100644 index 0000000000..3f922d4647 --- /dev/null +++ b/src/dfx-core/src/extension/url.rs @@ -0,0 +1,20 @@ +use url::Url; + +pub struct ExtensionJsonUrl(Url); + +impl ExtensionJsonUrl { + pub fn registered(name: &str) -> Result { + let s = format!( + "https://raw.githubusercontent.com/dfinity/dfx-extensions/main/extensions/{name}/extension.json" + ); + Url::parse(&s).map(ExtensionJsonUrl) + } + + pub fn to_dependencies_json(&self) -> Result { + self.as_url().join("dependencies.json") + } + + pub fn as_url(&self) -> &Url { + &self.0 + } +} diff --git a/src/dfx/src/commands/extension/install.rs b/src/dfx/src/commands/extension/install.rs index f1c26a8f60..2cdc9c9826 100644 --- a/src/dfx/src/commands/extension/install.rs +++ b/src/dfx/src/commands/extension/install.rs @@ -5,6 +5,7 @@ use crate::lib::error::DfxResult; use anyhow::bail; use clap::Parser; use clap::Subcommand; +use dfx_core::extension::url::ExtensionJsonUrl; use semver::Version; use tokio::runtime::Runtime; @@ -31,11 +32,14 @@ pub fn exec(env: &dyn Environment, opts: InstallOpts) -> DfxResult<()> { bail!("Extension '{}' cannot be installed because it conflicts with an existing command. Consider using '--install-as' flag to install this extension under different name.", opts.name) } + let url = ExtensionJsonUrl::registered(&opts.name)?; + let runtime = Runtime::new().expect("Unable to create a runtime"); - runtime.block_on(async { + let installed_version = runtime.block_on(async { mgr.install_extension( &opts.name, + &url, opts.install_as.as_deref(), opts.version.as_ref(), ) @@ -43,7 +47,7 @@ pub fn exec(env: &dyn Environment, opts: InstallOpts) -> DfxResult<()> { })?; spinner.finish_with_message( format!( - "Extension '{}' installed successfully{}", + "Extension '{}' version {installed_version} installed successfully{}", opts.name, if let Some(install_as) = opts.install_as { format!(", and is available as '{}'", install_as)