From 350a16d51c37200d4d4c22debb2243c8bcbe51d3 Mon Sep 17 00:00:00 2001 From: Tushar Sadhwani Date: Wed, 26 Jun 2024 12:03:49 +0530 Subject: [PATCH] Faster, and offline venvs (#22) --- install.bat | 3 +- install.ps1 | 3 +- install.sh | 15 +++++-- src/yen/__init__.py | 40 ++++++++++++++----- yen-rs/src/commands/create.rs | 41 +++++++++++++++++--- yen-rs/src/commands/install.rs | 19 ++------- yen-rs/src/github.rs | 19 ++++++--- yen-rs/src/main.rs | 1 + yen-rs/src/utils.rs | 71 +++++++++++++++++++++++++++++----- 9 files changed, 161 insertions(+), 51 deletions(-) diff --git a/install.bat b/install.bat index 2741010..064bcb2 100644 --- a/install.bat +++ b/install.bat @@ -10,8 +10,9 @@ mkdir "%yenpath%" 2>nul REM Download yen executable and save it to the .yen\bin directory SET "download_url=https://github.com/tusharsadhwani/yen/releases/latest/download/yen-rs-x86_64-pc-windows-msvc.exe" curl -SL --progress-bar "%download_url%" --output "%yenpath%\yen.exe" -REM Download userpath too +REM Download userpath and microvenv too curl -SL --progress-bar "https://yen.tushar.lol/userpath.pyz" --output "%yenpath%\userpath.pyz" +curl -SL --progress-bar "https://yen.tushar.lol/microvenv.py" --output "%yenpath%\microvenv.py" REM Get the user's PATH without the system-wide PATH for /f "skip=2 tokens=2,*" %%A in ('reg query HKCU\Environment /v PATH') do ( diff --git a/install.ps1 b/install.ps1 index 324f6e5..8f5a663 100644 --- a/install.ps1 +++ b/install.ps1 @@ -9,8 +9,9 @@ if (-not (Test-Path $yenpath)) { # Download yen executable and save it to the .yen\bin directory $downloadUrl = "https://github.com/tusharsadhwani/yen/releases/latest/download/yen-rs-x86_64-pc-windows-msvc.exe" Invoke-WebRequest -Uri $downloadUrl -OutFile "$yenpath\yen.exe" -# Download userpath too +# Download userpath and microvenv too Invoke-WebRequest -Uri "https://yen.tushar.lol/userpath.pyz" -OutFile "$yenpath\userpath.pyz" +Invoke-WebRequest -Uri "https://yen.tushar.lol/microvenv.py" -OutFile "$yenpath\microvenv.py" # Get the user's PATH without the system-wide PATH $userPath = (Get-ItemProperty -Path 'HKCU:\Environment' -Name 'Path').Path diff --git a/install.sh b/install.sh index 1e4b18f..8535739 100644 --- a/install.sh +++ b/install.sh @@ -46,9 +46,9 @@ fi # Move yen to the install directory mkdir -p "$INSTALL_DIR" -cp "$TEMP_FILE" "$INSTALL_DIR/yen" +mv "$TEMP_FILE" "$INSTALL_DIR/yen" -# Download userpath too +# Download userpath and microvenv too USERPATH_URL="https://yen.tushar.lol/userpath.pyz" HTTP_CODE=$(curl -SL --progress-bar "$USERPATH_URL" --output "$TEMP_FILE" --write-out "%{http_code}") if [ ${HTTP_CODE} -lt 200 ] || [ ${HTTP_CODE} -gt 299 ]; then @@ -56,7 +56,16 @@ if [ ${HTTP_CODE} -lt 200 ] || [ ${HTTP_CODE} -gt 299 ]; then exit 1 fi mkdir -p "$INSTALL_DIR" -cp "$TEMP_FILE" "$INSTALL_DIR/userpath.pyz" +mv "$TEMP_FILE" "$INSTALL_DIR/userpath.pyz" + +MICROVENV_URL="https://yen.tushar.lol/microvenv.py" +HTTP_CODE=$(curl -SL --progress-bar "$MICROVENV_URL" --output "$TEMP_FILE" --write-out "%{http_code}") +if [ ${HTTP_CODE} -lt 200 ] || [ ${HTTP_CODE} -gt 299 ]; then + echo "error: '${MICROVENV_URL}' is not available" + exit 1 +fi +mkdir -p "$INSTALL_DIR" +mv "$TEMP_FILE" "$INSTALL_DIR/microvenv.py" update_shell() { diff --git a/src/yen/__init__.py b/src/yen/__init__.py index 2297f27..dba11c0 100644 --- a/src/yen/__init__.py +++ b/src/yen/__init__.py @@ -26,6 +26,7 @@ ) USERPATH_PATH = os.path.join(YEN_BIN_PATH, "userpath.pyz") +MICROVENV_PATH = os.path.join(YEN_BIN_PATH, "microvenv.py") DEFAULT_PYTHON_VERSION = "3.12" @@ -62,6 +63,15 @@ def _ensure_userpath() -> None: urlretrieve("http://yen.tushar.lol/userpath.pyz", filename=USERPATH_PATH) +def _ensure_microvenv() -> None: + """Downloads `microvenv.py`, if it doesn't exist in `YEN_BIN_PATH`.""" + if os.path.exists(MICROVENV_PATH): + return + + os.makedirs(YEN_BIN_PATH, exist_ok=True) + urlretrieve("http://yen.tushar.lol/microvenv.py", filename=MICROVENV_PATH) + + def find_or_download_python() -> str: """ Finds and returns any Python binary from `PYTHON_INSTALLS_PATH`. @@ -100,14 +110,16 @@ def ensure_python(python_version: str) -> tuple[str, str]: """Checks if given Python version exists locally. If not, downloads it.""" os.makedirs(PYTHON_INSTALLS_PATH, exist_ok=True) + for python_folder_name in os.listdir(PYTHON_INSTALLS_PATH): + python_folder = os.path.join(PYTHON_INSTALLS_PATH, python_folder_name) + if python_folder_name.startswith(python_version): + # already installed + python_bin_path = _python_bin_path(python_folder) + return python_folder_name, python_bin_path + python_version, download_link = resolve_python_version(python_version) download_directory = os.path.join(PYTHON_INSTALLS_PATH, python_version) - python_bin_path = _python_bin_path(download_directory) - if os.path.exists(python_bin_path): - # already installed - return python_version, python_bin_path - os.makedirs(download_directory, exist_ok=True) downloaded_filepath = download( download_link, @@ -131,15 +143,25 @@ def ensure_python(python_version: str) -> tuple[str, str]: tar.extractall(download_directory) os.remove(downloaded_filepath) - assert os.path.exists(python_bin_path) + python_bin_path = _python_bin_path(download_directory) + assert os.path.exists(python_bin_path) return python_version, python_bin_path def create_venv(python_bin_path: str, venv_path: str) -> None: - # TODO: bundle microvenv.pyz as a dependency, venv is genuinely too slow - # microvenv doesn't support windows, fallback to venv for that. teehee. - subprocess.run([python_bin_path, "-m", "venv", venv_path], check=True) + if platform.system() == "Windows": + subprocess.run([python_bin_path, "-m", "venv", venv_path], check=True) + return + + _ensure_microvenv() + subprocess.run([python_bin_path, MICROVENV_PATH, venv_path], check=True) + venv_python_path = _venv_binary_path("python", venv_path) + subprocess.run( + [venv_python_path, "-m", "ensurepip"], + check=True, + capture_output=True, + ) def _venv_binary_path(binary_name: str, venv_path: str) -> str: diff --git a/yen-rs/src/commands/create.rs b/yen-rs/src/commands/create.rs index ed22159..8119297 100644 --- a/yen-rs/src/commands/create.rs +++ b/yen-rs/src/commands/create.rs @@ -3,7 +3,11 @@ use std::{path::PathBuf, process::Command}; use clap::Parser; use miette::IntoDiagnostic; -use crate::{github::Version, utils::ensure_python}; +use crate::{ + github::Version, + utils::{_ensure_microvenv, _venv_binary_path, ensure_python, IS_WINDOWS}, + MICROVENV_PATH, +}; /// Create venv with python version #[derive(Parser, Debug)] @@ -23,10 +27,21 @@ pub async fn create_env(python_bin_path: PathBuf, venv_path: &PathBuf) -> miette miette::bail!("Error: {} already exists!", venv_path.to_string_lossy()); } - let stdout = Command::new(format!("{}", python_bin_path.to_string_lossy())) - .args(["-m", "venv", &format!("{}", venv_path.to_string_lossy())]) - .output() - .into_diagnostic()?; + let stdout = if IS_WINDOWS { + Command::new(format!("{}", python_bin_path.to_string_lossy())) + .args(["-m", "venv", &format!("{}", venv_path.to_string_lossy())]) + .output() + .into_diagnostic()? + } else { + _ensure_microvenv().await?; + Command::new(format!("{}", python_bin_path.to_string_lossy())) + .args([ + &MICROVENV_PATH.to_string_lossy().into_owned(), + &venv_path.to_string_lossy().into_owned(), + ]) + .output() + .into_diagnostic()? + }; if !stdout.status.success() { miette::bail!(format!( @@ -36,6 +51,22 @@ pub async fn create_env(python_bin_path: PathBuf, venv_path: &PathBuf) -> miette )); } + if !IS_WINDOWS { + let venv_python_path = _venv_binary_path("python", venv_path); + let stdout = Command::new(format!("{}", venv_python_path.to_string_lossy())) + .args(["-m", "ensurepip"]) + .output() + .into_diagnostic()?; + + if !stdout.status.success() { + miette::bail!(format!( + "Error: unable to run ensurepip!\nStdout: {}\nStderr: {}", + String::from_utf8_lossy(&stdout.stdout), + String::from_utf8_lossy(&stdout.stderr), + )); + } + } + Ok(()) } diff --git a/yen-rs/src/commands/install.rs b/yen-rs/src/commands/install.rs index 31d3877..f776c19 100644 --- a/yen-rs/src/commands/install.rs +++ b/yen-rs/src/commands/install.rs @@ -5,17 +5,14 @@ use miette::IntoDiagnostic; use crate::{ github::Version, - utils::{_ensure_userpath, ensure_python, find_or_download_python}, + utils::{ + _ensure_userpath, _venv_binary_path, ensure_python, find_or_download_python, IS_WINDOWS, + }, DEFAULT_PYTHON_VERSION, PACKAGE_INSTALLS_PATH, USERPATH_PATH, }; use super::create::create_env; -#[cfg(target_os = "windows")] -const IS_WINDOWS: bool = true; -#[cfg(not(target_os = "windows"))] -const IS_WINDOWS: bool = false; - /// Install a Python package in an isolated environment. #[derive(Parser, Debug)] pub struct Args { @@ -101,16 +98,6 @@ async fn check_path(path: PathBuf) -> miette::Result<()> { Ok(()) } -fn _venv_binary_path(binary_name: &str, venv_path: &std::path::PathBuf) -> std::path::PathBuf { - let venv_bin_path = venv_path.join(if IS_WINDOWS { "Scripts" } else { "bin" }); - let binary_path = venv_bin_path.join(if IS_WINDOWS { - format!("{binary_name}.exe") - } else { - binary_name.to_string() - }); - return binary_path; -} - pub async fn install_package( package_name: &str, python_bin_path: std::path::PathBuf, diff --git a/yen-rs/src/github.rs b/yen-rs/src/github.rs index 6ed87f2..fd97e35 100644 --- a/yen-rs/src/github.rs +++ b/yen-rs/src/github.rs @@ -118,7 +118,7 @@ impl MachineSuffix { const FALLBACK_RESPONSE_BYTES: &[u8] = include_bytes!("../../src/yen/fallback_release_data.json"); -async fn get_latest_python_release() -> miette::Result> { +async fn get_release_json() -> miette::Result { let response = YEN_CLIENT .get(*GITHUB_API_URL) .send() @@ -129,20 +129,27 @@ async fn get_latest_python_release() -> miette::Result> { // Log the response body if the status is not successful let status_code = response.status().as_u16(); let success = response.status().is_success(); - let mut body = response.text().await.into_diagnostic()?; + let body = response.text().await.into_diagnostic()?; if !success { log::error!("Error response: {}\nStatus Code: {}", body, status_code); - let fallback_response = String::from_utf8_lossy(FALLBACK_RESPONSE_BYTES).into_owned(); - body = fallback_response; + miette::bail!("Failed to fetch fallback data"); } + Ok(body) +} + +async fn get_latest_python_release() -> miette::Result> { + let json = get_release_json() + .await + .unwrap_or(String::from_utf8_lossy(FALLBACK_RESPONSE_BYTES).into_owned()); + // Attempt to parse the JSON - let github_resp = match serde_json::from_str::(&body) { + let github_resp = match serde_json::from_str::(&json) { Ok(data) => data, Err(err) => { // Log the error and response body in case of JSON decoding failure log::error!("Error decoding JSON: {}", err); - log::error!("Response body: {}", body); + log::error!("Response body: {}", json); miette::bail!("JSON decoding error, check the logs for more info."); } }; diff --git a/yen-rs/src/main.rs b/yen-rs/src/main.rs index 4d595da..67ab0b3 100644 --- a/yen-rs/src/main.rs +++ b/yen-rs/src/main.rs @@ -44,6 +44,7 @@ lazy_static! { .expect("Failed to turn YEN_BIN_PATH into absolute") }; static ref USERPATH_PATH: PathBuf = YEN_BIN_PATH.join("userpath.pyz"); + static ref MICROVENV_PATH: PathBuf = YEN_BIN_PATH.join("microvenv.py"); static ref YEN_CLIENT: Client = yen_client(); } diff --git a/yen-rs/src/utils.rs b/yen-rs/src/utils.rs index af8b7bf..d5a4909 100644 --- a/yen-rs/src/utils.rs +++ b/yen-rs/src/utils.rs @@ -16,7 +16,8 @@ use tar::Archive; use crate::{ github::{resolve_python_version, Version}, - DEFAULT_PYTHON_VERSION, PYTHON_INSTALLS_PATH, USERPATH_PATH, YEN_BIN_PATH, YEN_CLIENT, + DEFAULT_PYTHON_VERSION, MICROVENV_PATH, PYTHON_INSTALLS_PATH, USERPATH_PATH, YEN_BIN_PATH, + YEN_CLIENT, }; #[cfg(target_os = "linux")] @@ -25,37 +26,52 @@ use crate::MUSL; #[cfg(target_os = "linux")] use std::io::Read; +#[cfg(target_os = "windows")] +pub const IS_WINDOWS: bool = true; +#[cfg(not(target_os = "windows"))] +pub const IS_WINDOWS: bool = false; + pub async fn ensure_python(version: Version) -> miette::Result<(Version, PathBuf)> { if !PYTHON_INSTALLS_PATH.exists() { fs::create_dir(PYTHON_INSTALLS_PATH.to_path_buf()).into_diagnostic()?; } + for path in std::fs::read_dir(PYTHON_INSTALLS_PATH.to_path_buf()).into_diagnostic()? { + let Ok(python_folder) = path else { + continue; + }; - let (version, link) = resolve_python_version(version).await?; - - let download_dir = PYTHON_INSTALLS_PATH.join(version.to_string()); - - let python_bin_path = _python_bin_path(&download_dir); - if python_bin_path.exists() { + let resolved_python_version = python_folder.file_name().to_string_lossy().into_owned(); + if !resolved_python_version.starts_with(&version.to_string()) { + continue; + } + let Ok(version) = Version::from_str(&resolved_python_version) else { + continue; + }; + let python_bin_path = _python_bin_path(&python_folder.path()); return Ok((version, python_bin_path)); } + let (version, link) = resolve_python_version(version).await?; + let download_dir = PYTHON_INSTALLS_PATH.join(version.to_string()); if !download_dir.exists() { fs::create_dir_all(&download_dir).into_diagnostic()?; } let downloaded_file = download(link.as_str(), &download_dir).await?; - let file = File::open(downloaded_file).into_diagnostic()?; - Archive::new(GzDecoder::new(file)) - .unpack(download_dir) + .unpack(&download_dir) .into_diagnostic()?; + let python_bin_path = _python_bin_path(&download_dir); Ok((version, python_bin_path)) } /// Finds and returns any Python binary from `PYTHON_INSTALLS_PATH`. /// If no Pythons exist, downloads the default version and returns that. pub async fn find_or_download_python() -> miette::Result { + if !PYTHON_INSTALLS_PATH.exists() { + fs::create_dir(PYTHON_INSTALLS_PATH.to_path_buf()).into_diagnostic()?; + } for path in std::fs::read_dir(PYTHON_INSTALLS_PATH.to_path_buf()).into_diagnostic()? { let Ok(python_folder) = path else { continue; @@ -95,6 +111,30 @@ pub async fn _ensure_userpath() -> miette::Result<()> { Ok(()) } +/// Downloads `microvenv.py`, if it doesn't exist in `YEN_BIN_PATH`. +pub async fn _ensure_microvenv() -> miette::Result<()> { + if MICROVENV_PATH.exists() { + return Ok(()); + } + + if !YEN_BIN_PATH.exists() { + std::fs::create_dir_all(YEN_BIN_PATH.to_path_buf()).into_diagnostic()?; + } + + let microvenv_content = YEN_CLIENT + .get("https://yen.tushar.lol/microvenv.py") + .send() + .await + .into_diagnostic()? + .bytes() + .await + .into_diagnostic()?; + + std::fs::write(MICROVENV_PATH.to_path_buf(), microvenv_content).into_diagnostic()?; + Ok(()) +} + +/// Returns the path to the Python executable inside a downloaded Python pub fn _python_bin_path(download_dir: &PathBuf) -> PathBuf { #[cfg(target_os = "windows")] let python_bin_path = download_dir.join("python/python.exe"); @@ -103,6 +143,17 @@ pub fn _python_bin_path(download_dir: &PathBuf) -> PathBuf { python_bin_path } +/// Returns the path to a binary inside a venv +pub fn _venv_binary_path(binary_name: &str, venv_path: &std::path::PathBuf) -> std::path::PathBuf { + let venv_bin_path = venv_path.join(if IS_WINDOWS { "Scripts" } else { "bin" }); + let binary_path = venv_bin_path.join(if IS_WINDOWS { + format!("{binary_name}.exe") + } else { + binary_name.to_string() + }); + return binary_path; +} + pub async fn download(link: &str, path: &Path) -> miette::Result { let filepath = path.join( link.split('/')