Faster, and offline venvs (#22)

install.bat

@@ -10,8 +10,9 @@ mkdir "%yenpath%" 2>nul
 REM Download yen executable and save it to the .yen\bin directory
 SET "download_url=https://github.com/tusharsadhwani/yen/releases/latest/download/yen-rs-x86_64-pc-windows-msvc.exe"
 curl -SL --progress-bar "%download_url%" --output "%yenpath%\yen.exe"
-REM Download userpath too
+REM Download userpath and microvenv too
 curl -SL --progress-bar "https://yen.tushar.lol/userpath.pyz" --output "%yenpath%\userpath.pyz"
+curl -SL --progress-bar "https://yen.tushar.lol/microvenv.py" --output "%yenpath%\microvenv.py"
 
 REM Get the user's PATH without the system-wide PATH
 for /f "skip=2 tokens=2,*" %%A in ('reg query HKCU\Environment /v PATH') do (

install.ps1

@@ -9,8 +9,9 @@ if (-not (Test-Path $yenpath)) {
 # Download yen executable and save it to the .yen\bin directory
 $downloadUrl = "https://github.com/tusharsadhwani/yen/releases/latest/download/yen-rs-x86_64-pc-windows-msvc.exe"
 Invoke-WebRequest -Uri $downloadUrl -OutFile "$yenpath\yen.exe"
-# Download userpath too
+# Download userpath and microvenv too
 Invoke-WebRequest -Uri "https://yen.tushar.lol/userpath.pyz" -OutFile "$yenpath\userpath.pyz"
+Invoke-WebRequest -Uri "https://yen.tushar.lol/microvenv.py" -OutFile "$yenpath\microvenv.py"
 
 # Get the user's PATH without the system-wide PATH
 $userPath = (Get-ItemProperty -Path 'HKCU:\Environment' -Name 'Path').Path

install.sh

@@ -46,17 +46,26 @@ fi
 
 # Move yen to the install directory
 mkdir -p "$INSTALL_DIR"
-cp "$TEMP_FILE" "$INSTALL_DIR/yen"
+mv "$TEMP_FILE" "$INSTALL_DIR/yen"
 
-# Download userpath too
+# Download userpath and microvenv too
 USERPATH_URL="https://yen.tushar.lol/userpath.pyz"
 HTTP_CODE=$(curl -SL --progress-bar "$USERPATH_URL" --output "$TEMP_FILE" --write-out "%{http_code}")
 if [ ${HTTP_CODE} -lt 200 ] || [ ${HTTP_CODE} -gt 299 ]; then
   echo "error: '${USERPATH_URL}' is not available"
   exit 1
 fi
 mkdir -p "$INSTALL_DIR"
-cp "$TEMP_FILE" "$INSTALL_DIR/userpath.pyz"
+mv "$TEMP_FILE" "$INSTALL_DIR/userpath.pyz"
+
+MICROVENV_URL="https://yen.tushar.lol/microvenv.py"
+HTTP_CODE=$(curl -SL --progress-bar "$MICROVENV_URL" --output "$TEMP_FILE" --write-out "%{http_code}")
+if [ ${HTTP_CODE} -lt 200 ] || [ ${HTTP_CODE} -gt 299 ]; then
+  echo "error: '${MICROVENV_URL}' is not available"
+  exit 1
+fi
+mkdir -p "$INSTALL_DIR"
+mv "$TEMP_FILE" "$INSTALL_DIR/microvenv.py"
 
 
 update_shell() {

src/yen/__init__.py

@@ -26,6 +26,7 @@
 )
 
 USERPATH_PATH = os.path.join(YEN_BIN_PATH, "userpath.pyz")
+MICROVENV_PATH = os.path.join(YEN_BIN_PATH, "microvenv.py")
 
 DEFAULT_PYTHON_VERSION = "3.12"
 
@@ -62,6 +63,15 @@ def _ensure_userpath() -> None:
     urlretrieve("http://yen.tushar.lol/userpath.pyz", filename=USERPATH_PATH)
 
 
+def _ensure_microvenv() -> None:
+    """Downloads `microvenv.py`, if it doesn't exist in `YEN_BIN_PATH`."""
+    if os.path.exists(MICROVENV_PATH):
+        return
+
+    os.makedirs(YEN_BIN_PATH, exist_ok=True)
+    urlretrieve("http://yen.tushar.lol/microvenv.py", filename=MICROVENV_PATH)
+
+
 def find_or_download_python() -> str:
     """
     Finds and returns any Python binary from `PYTHON_INSTALLS_PATH`.
@@ -100,14 +110,16 @@ def ensure_python(python_version: str) -> tuple[str, str]:
     """Checks if given Python version exists locally. If not, downloads it."""
     os.makedirs(PYTHON_INSTALLS_PATH, exist_ok=True)
 
+    for python_folder_name in os.listdir(PYTHON_INSTALLS_PATH):
+        python_folder = os.path.join(PYTHON_INSTALLS_PATH, python_folder_name)
+        if python_folder_name.startswith(python_version):
+            # already installed
+            python_bin_path = _python_bin_path(python_folder)
+            return python_folder_name, python_bin_path
+
     python_version, download_link = resolve_python_version(python_version)
     download_directory = os.path.join(PYTHON_INSTALLS_PATH, python_version)
 
-    python_bin_path = _python_bin_path(download_directory)
-    if os.path.exists(python_bin_path):
-        # already installed
-        return python_version, python_bin_path
-
     os.makedirs(download_directory, exist_ok=True)
     downloaded_filepath = download(
         download_link,
@@ -131,15 +143,25 @@ def ensure_python(python_version: str) -> tuple[str, str]:
         tar.extractall(download_directory)
 
     os.remove(downloaded_filepath)
-    assert os.path.exists(python_bin_path)
 
+    python_bin_path = _python_bin_path(download_directory)
+    assert os.path.exists(python_bin_path)
     return python_version, python_bin_path
 
 
 def create_venv(python_bin_path: str, venv_path: str) -> None:
-    # TODO: bundle microvenv.pyz as a dependency, venv is genuinely too slow
-    # microvenv doesn't support windows, fallback to venv for that. teehee.
-    subprocess.run([python_bin_path, "-m", "venv", venv_path], check=True)
+    if platform.system() == "Windows":
+        subprocess.run([python_bin_path, "-m", "venv", venv_path], check=True)
+        return
+
+    _ensure_microvenv()
+    subprocess.run([python_bin_path, MICROVENV_PATH, venv_path], check=True)
+    venv_python_path = _venv_binary_path("python", venv_path)
+    subprocess.run(
+        [venv_python_path, "-m", "ensurepip"],
+        check=True,
+        capture_output=True,
+    )
 
 
 def _venv_binary_path(binary_name: str, venv_path: str) -> str:

yen-rs/src/commands/create.rs

@@ -3,7 +3,11 @@ use std::{path::PathBuf, process::Command};
 use clap::Parser;
 use miette::IntoDiagnostic;
 
-use crate::{github::Version, utils::ensure_python};
+use crate::{
+    github::Version,
+    utils::{_ensure_microvenv, _venv_binary_path, ensure_python, IS_WINDOWS},
+    MICROVENV_PATH,
+};
 
 /// Create venv with python version
 #[derive(Parser, Debug)]
@@ -23,10 +27,21 @@ pub async fn create_env(python_bin_path: PathBuf, venv_path: &PathBuf) -> miette
         miette::bail!("Error: {} already exists!", venv_path.to_string_lossy());
     }
 
-    let stdout = Command::new(format!("{}", python_bin_path.to_string_lossy()))
-        .args(["-m", "venv", &format!("{}", venv_path.to_string_lossy())])
-        .output()
-        .into_diagnostic()?;
+    let stdout = if IS_WINDOWS {
+        Command::new(format!("{}", python_bin_path.to_string_lossy()))
+            .args(["-m", "venv", &format!("{}", venv_path.to_string_lossy())])
+            .output()
+            .into_diagnostic()?
+    } else {
+        _ensure_microvenv().await?;
+        Command::new(format!("{}", python_bin_path.to_string_lossy()))
+            .args([
+                &MICROVENV_PATH.to_string_lossy().into_owned(),
+                &venv_path.to_string_lossy().into_owned(),
+            ])
+            .output()
+            .into_diagnostic()?
+    };
 
     if !stdout.status.success() {
         miette::bail!(format!(
@@ -36,6 +51,22 @@ pub async fn create_env(python_bin_path: PathBuf, venv_path: &PathBuf) -> miette
         ));
     }
 
+    if !IS_WINDOWS {
+        let venv_python_path = _venv_binary_path("python", venv_path);
+        let stdout = Command::new(format!("{}", venv_python_path.to_string_lossy()))
+            .args(["-m", "ensurepip"])
+            .output()
+            .into_diagnostic()?;
+
+        if !stdout.status.success() {
+            miette::bail!(format!(
+                "Error: unable to run ensurepip!\nStdout: {}\nStderr: {}",
+                String::from_utf8_lossy(&stdout.stdout),
+                String::from_utf8_lossy(&stdout.stderr),
+            ));
+        }
+    }
+
     Ok(())
 }
 

yen-rs/src/commands/install.rs

@@ -5,17 +5,14 @@ use miette::IntoDiagnostic;
 
 use crate::{
     github::Version,
-    utils::{_ensure_userpath, ensure_python, find_or_download_python},
+    utils::{
+        _ensure_userpath, _venv_binary_path, ensure_python, find_or_download_python, IS_WINDOWS,
+    },
     DEFAULT_PYTHON_VERSION, PACKAGE_INSTALLS_PATH, USERPATH_PATH,
 };
 
 use super::create::create_env;
 
-#[cfg(target_os = "windows")]
-const IS_WINDOWS: bool = true;
-#[cfg(not(target_os = "windows"))]
-const IS_WINDOWS: bool = false;
-
 /// Install a Python package in an isolated environment.
 #[derive(Parser, Debug)]
 pub struct Args {
@@ -101,16 +98,6 @@ async fn check_path(path: PathBuf) -> miette::Result<()> {
     Ok(())
 }
 
-fn _venv_binary_path(binary_name: &str, venv_path: &std::path::PathBuf) -> std::path::PathBuf {
-    let venv_bin_path = venv_path.join(if IS_WINDOWS { "Scripts" } else { "bin" });
-    let binary_path = venv_bin_path.join(if IS_WINDOWS {
-        format!("{binary_name}.exe")
-    } else {
-        binary_name.to_string()
-    });
-    return binary_path;
-}
-
 pub async fn install_package(
     package_name: &str,
     python_bin_path: std::path::PathBuf,

yen-rs/src/github.rs

@@ -118,7 +118,7 @@ impl MachineSuffix {
 
 const FALLBACK_RESPONSE_BYTES: &[u8] = include_bytes!("../../src/yen/fallback_release_data.json");
 
-async fn get_latest_python_release() -> miette::Result<Vec<String>> {
+async fn get_release_json() -> miette::Result<String> {
     let response = YEN_CLIENT
         .get(*GITHUB_API_URL)
         .send()
@@ -129,20 +129,27 @@ async fn get_latest_python_release() -> miette::Result<Vec<String>> {
     // Log the response body if the status is not successful
     let status_code = response.status().as_u16();
     let success = response.status().is_success();
-    let mut body = response.text().await.into_diagnostic()?;
+    let body = response.text().await.into_diagnostic()?;
     if !success {
         log::error!("Error response: {}\nStatus Code: {}", body, status_code);
-        let fallback_response = String::from_utf8_lossy(FALLBACK_RESPONSE_BYTES).into_owned();
-        body = fallback_response;
+        miette::bail!("Failed to fetch fallback data");
     }
 
+    Ok(body)
+}
+
+async fn get_latest_python_release() -> miette::Result<Vec<String>> {
+    let json = get_release_json()
+        .await
+        .unwrap_or(String::from_utf8_lossy(FALLBACK_RESPONSE_BYTES).into_owned());
+
     // Attempt to parse the JSON
-    let github_resp = match serde_json::from_str::<GithubResp>(&body) {
+    let github_resp = match serde_json::from_str::<GithubResp>(&json) {
         Ok(data) => data,
         Err(err) => {
             // Log the error and response body in case of JSON decoding failure
             log::error!("Error decoding JSON: {}", err);
-            log::error!("Response body: {}", body);
+            log::error!("Response body: {}", json);
             miette::bail!("JSON decoding error, check the logs for more info.");
         }
     };

yen-rs/src/main.rs

@@ -44,6 +44,7 @@ lazy_static! {
         .expect("Failed to turn YEN_BIN_PATH into absolute")
     };
     static ref USERPATH_PATH: PathBuf = YEN_BIN_PATH.join("userpath.pyz");
+    static ref MICROVENV_PATH: PathBuf = YEN_BIN_PATH.join("microvenv.py");
     static ref YEN_CLIENT: Client = yen_client();
 }
 

yen-rs/src/utils.rs

@@ -16,7 +16,8 @@ use tar::Archive;
 
 use crate::{
     github::{resolve_python_version, Version},
-    DEFAULT_PYTHON_VERSION, PYTHON_INSTALLS_PATH, USERPATH_PATH, YEN_BIN_PATH, YEN_CLIENT,
+    DEFAULT_PYTHON_VERSION, MICROVENV_PATH, PYTHON_INSTALLS_PATH, USERPATH_PATH, YEN_BIN_PATH,
+    YEN_CLIENT,
 };
 
 #[cfg(target_os = "linux")]
@@ -25,37 +26,52 @@ use crate::MUSL;
 #[cfg(target_os = "linux")]
 use std::io::Read;
 
+#[cfg(target_os = "windows")]
+pub const IS_WINDOWS: bool = true;
+#[cfg(not(target_os = "windows"))]
+pub const IS_WINDOWS: bool = false;
+
 pub async fn ensure_python(version: Version) -> miette::Result<(Version, PathBuf)> {
     if !PYTHON_INSTALLS_PATH.exists() {
         fs::create_dir(PYTHON_INSTALLS_PATH.to_path_buf()).into_diagnostic()?;
     }
+    for path in std::fs::read_dir(PYTHON_INSTALLS_PATH.to_path_buf()).into_diagnostic()? {
+        let Ok(python_folder) = path else {
+            continue;
+        };
 
-    let (version, link) = resolve_python_version(version).await?;
-
-    let download_dir = PYTHON_INSTALLS_PATH.join(version.to_string());
-
-    let python_bin_path = _python_bin_path(&download_dir);
-    if python_bin_path.exists() {
+        let resolved_python_version = python_folder.file_name().to_string_lossy().into_owned();
+        if !resolved_python_version.starts_with(&version.to_string()) {
+            continue;
+        }
+        let Ok(version) = Version::from_str(&resolved_python_version) else {
+            continue;
+        };
+        let python_bin_path = _python_bin_path(&python_folder.path());
         return Ok((version, python_bin_path));
     }
 
+    let (version, link) = resolve_python_version(version).await?;
+    let download_dir = PYTHON_INSTALLS_PATH.join(version.to_string());
     if !download_dir.exists() {
         fs::create_dir_all(&download_dir).into_diagnostic()?;
     }
     let downloaded_file = download(link.as_str(), &download_dir).await?;
-
     let file = File::open(downloaded_file).into_diagnostic()?;
-
     Archive::new(GzDecoder::new(file))
-        .unpack(download_dir)
+        .unpack(&download_dir)
         .into_diagnostic()?;
 
+    let python_bin_path = _python_bin_path(&download_dir);
     Ok((version, python_bin_path))
 }
 
 /// Finds and returns any Python binary from `PYTHON_INSTALLS_PATH`.
 /// If no Pythons exist, downloads the default version and returns that.
 pub async fn find_or_download_python() -> miette::Result<PathBuf> {
+    if !PYTHON_INSTALLS_PATH.exists() {
+        fs::create_dir(PYTHON_INSTALLS_PATH.to_path_buf()).into_diagnostic()?;
+    }
     for path in std::fs::read_dir(PYTHON_INSTALLS_PATH.to_path_buf()).into_diagnostic()? {
         let Ok(python_folder) = path else {
             continue;
@@ -95,6 +111,30 @@ pub async fn _ensure_userpath() -> miette::Result<()> {
     Ok(())
 }
 
+/// Downloads `microvenv.py`, if it doesn't exist in `YEN_BIN_PATH`.
+pub async fn _ensure_microvenv() -> miette::Result<()> {
+    if MICROVENV_PATH.exists() {
+        return Ok(());
+    }
+
+    if !YEN_BIN_PATH.exists() {
+        std::fs::create_dir_all(YEN_BIN_PATH.to_path_buf()).into_diagnostic()?;
+    }
+
+    let microvenv_content = YEN_CLIENT
+        .get("https://yen.tushar.lol/microvenv.py")
+        .send()
+        .await
+        .into_diagnostic()?
+        .bytes()
+        .await
+        .into_diagnostic()?;
+
+    std::fs::write(MICROVENV_PATH.to_path_buf(), microvenv_content).into_diagnostic()?;
+    Ok(())
+}
+
+/// Returns the path to the Python executable inside a downloaded Python
 pub fn _python_bin_path(download_dir: &PathBuf) -> PathBuf {
     #[cfg(target_os = "windows")]
     let python_bin_path = download_dir.join("python/python.exe");
@@ -103,6 +143,17 @@ pub fn _python_bin_path(download_dir: &PathBuf) -> PathBuf {
     python_bin_path
 }
 
+/// Returns the path to a binary inside a venv
+pub fn _venv_binary_path(binary_name: &str, venv_path: &std::path::PathBuf) -> std::path::PathBuf {
+    let venv_bin_path = venv_path.join(if IS_WINDOWS { "Scripts" } else { "bin" });
+    let binary_path = venv_bin_path.join(if IS_WINDOWS {
+        format!("{binary_name}.exe")
+    } else {
+        binary_name.to_string()
+    });
+    return binary_path;
+}
+
 pub async fn download(link: &str, path: &Path) -> miette::Result<PathBuf> {
     let filepath = path.join(
         link.split('/')