feat: uv support, py 3.13 (#1216)

* feat: py 3.13 and uv support * test: uv * refactor: simplify variable assignment * style: formatting fix * style: docs lint fix * fix: adding snapshot * fix: bump patch py version * fix: main.py should be selected first * feat: update command in generate_plan_tests snapshot * fix: update Python Nixpkgs archive hash and pkg name - Changed the PYTHON_NIXPKGS_ARCHIVE to a new hash. - Updated PostgreSQL package to require the .dev variant for compatibility with psycopg2. - Added comments to clarify the changes for future reference. * fix: update snapshot kind and PostgreSQL dependency - Add `snapshot_kind: text` to snapshots - Change `postgresql` to `postgresql.dev` in dependencies --------- Co-authored-by: Jake Runzer <[email protected]>
railwayapp · Nov 18, 2024 · 3077a7a · 3077a7a
1 parent 8a31920
commit 3077a7a
Show file tree

Hide file tree

Showing 11 changed files with 357 additions and 20 deletions.
diff --git a/docs/pages/docs/providers/python.md b/docs/pages/docs/providers/python.md
@@ -11,6 +11,8 @@ Python is detected if any of the following files are found
 - `pyproject.toml`
 - `Pipfile`
 
+A venv is created at `/opt/venv` and `PATH` is modified to use the venv python binary.
+
 ## Setup
 
 The following Python versions are available
@@ -21,12 +23,20 @@ The following Python versions are available
 - `3.10`
 - `3.11` (Default)
 - `3.12`
+- `3.13`
 
 The version can be overridden by
 
 - Setting the `NIXPACKS_PYTHON_VERSION` environment variable
 - Setting the version in a `.python-version` file
 - Setting the version in a `runtime.txt` file
+- Setting the version in a `.tool-versions` file
+
+You also specify the exact poetry, pdm, and uv versions:
+
+- The `NIXPACKS_POETRY_VERSION` environment variable or `poetry` in a `.tool-versions` file
+- The `NIXPACKS_PDM_VERSION` environment variable
+- The `NIXPACKS_UV_VERSION` environment variable or `uv` in a `.tool-versions` file
 
 ## Install
 
@@ -60,6 +70,12 @@ If `Pipfile` (w/ `Pipfile.lock`)
 PIPENV_VENV_IN_PROJECT=1 pipenv install --deploy
 ```
 
+if `uv.lock`:
+
+```
+uv sync --no-dev --frozen
+```
+
 ## Start
 
 if Django Application
@@ -85,6 +101,8 @@ python main.py
 These directories are cached between builds
 
 - Install: `~/.cache/pip`
+- Install: `~/.cache/uv`
+- Install: `~/.cache/pdm`
 
 ## Environment Variables
 

diff --git a/examples/python-uv/.python-version b/examples/python-uv/.python-version
@@ -0,0 +1 @@
+3.12
diff --git a/examples/python-uv/README.md b/examples/python-uv/README.md
diff --git a/examples/python-uv/main.py b/examples/python-uv/main.py
@@ -0,0 +1 @@
+print("Hello from Python-Uv")
diff --git a/examples/python-uv/pyproject.toml b/examples/python-uv/pyproject.toml
@@ -0,0 +1,14 @@
+[project]
+name = "python-uv"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "flask>=3.0.3",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.3.3",
+]
diff --git a/examples/python-uv/uv.lock b/examples/python-uv/uv.lock
diff --git a/src/providers/python.rs b/src/providers/python.rs
@@ -22,11 +22,15 @@ use super::{Provider, ProviderMetadata};
 const DEFAULT_PYTHON_PKG_NAME: &str = "python3";
 const POETRY_VERSION: &str = "1.3.1";
 const PDM_VERSION: &str = "2.13.3";
+const UV_VERSION: &str = "0.4.30";
+
+const VENV_LOCATION: &str = "/opt/venv";
+const UV_CACHE_DIR: &str = "/root/.cache/uv";
 const PIP_CACHE_DIR: &str = "/root/.cache/pip";
 const PDM_CACHE_DIR: &str = "/root/.cache/pdm";
 const DEFAULT_POETRY_PYTHON_PKG_NAME: &str = "python3";
 
-const PYTHON_NIXPKGS_ARCHIVE: &str = "bf446f08bff6814b569265bef8374cfdd3d8f0e0";
+const PYTHON_NIXPKGS_ARCHIVE: &str = "bc8f8d1be58e8c8383e683a06e1e1e57893fff87";
 const LEGACY_PYTHON_NIXPKGS_ARCHIVE: &str = "5148520bfab61f99fd25fb9ff7bfbb50dad3c9db";
 
 pub struct PythonProvider {}
@@ -92,13 +96,38 @@ impl Provider for PythonProvider {
                 version,
             )]));
         }
+
         if app.includes_file("pdm.lock") {
             plan.add_variables(EnvironmentVariables::from([(
                 "NIXPACKS_PDM_VERSION".to_string(),
                 PDM_VERSION.to_string(),
             )]));
         }
 
+        // uv version is not, as of 0.4.30, specified in the lock file or pyproject.toml
+        if app.includes_file("uv.lock") {
+            let mut version = UV_VERSION.to_string();
+
+            if app.includes_file(".tool-versions") {
+                let file_content = &app.read_file(".tool-versions")?;
+
+                if let Some(uv_version) =
+                    PythonProvider::parse_tool_versions_uv_version(file_content)?
+                {
+                    println!("Using uv version from .tool-versions: {uv_version}");
+                    version = uv_version;
+                }
+            }
+
+            plan.add_variables(EnvironmentVariables::from([
+                ("NIXPACKS_UV_VERSION".to_string(), version),
+                (
+                    "UV_PROJECT_ENVIRONMENT".to_string(),
+                    VENV_LOCATION.to_string(),
+                ),
+            ]));
+        }
+
         Ok(Some(plan))
     }
 }
@@ -140,7 +169,11 @@ impl PythonProvider {
 
         if PythonProvider::is_using_postgres(app, env)? {
             // Postgres requires postgresql and gcc on top of the original python packages
-            pkgs.append(&mut vec![Pkg::new("postgresql")]);
+
+            // .dev variant is required in order for pg_config to be available, which is needed by psycopg2
+            // the .dev variant requirement is caused by this change in nix pkgs:
+            // https://github.com/NixOS/nixpkgs/blob/43eac3c9e618c4114a3441b52949609ea2104670/pkgs/servers/sql/postgresql/pg_config.sh
+            pkgs.append(&mut vec![Pkg::new("postgresql.dev")]);
         }
 
         if PythonProvider::is_django(app, env)? && PythonProvider::is_using_mysql(app, env)? {
@@ -168,16 +201,15 @@ impl PythonProvider {
     }
 
     fn install(&self, app: &App, _env: &Environment) -> Result<Option<Phase>> {
-        let env_loc = "/opt/venv";
-        let create_env = format!("python -m venv --copies {env_loc}");
-        let activate_env = format!(". {env_loc}/bin/activate");
+        let create_env = format!("python -m venv --copies {VENV_LOCATION}");
+        let activate_env = format!(". {VENV_LOCATION}/bin/activate");
 
         if app.includes_file("requirements.txt") {
             let mut install_phase = Phase::install(Some(format!(
                 "{create_env} && {activate_env} && pip install -r requirements.txt"
             )));
 
-            install_phase.add_path(format!("{env_loc}/bin"));
+            install_phase.add_path(format!("{VENV_LOCATION}/bin"));
             install_phase.add_cache_directory(PIP_CACHE_DIR.to_string());
 
             return Ok(Some(install_phase));
@@ -188,7 +220,7 @@ impl PythonProvider {
                     "{create_env} && {activate_env} && {install_poetry} && poetry install --no-dev --no-interaction --no-ansi"
                 )));
 
-                install_phase.add_path(format!("{env_loc}/bin"));
+                install_phase.add_path(format!("{VENV_LOCATION}/bin"));
 
                 install_phase.add_cache_directory(PIP_CACHE_DIR.to_string());
 
@@ -199,19 +231,37 @@ impl PythonProvider {
                     "{create_env} && {activate_env} && {install_pdm} && pdm install --prod"
                 )));
 
-                install_phase.add_path(format!("{env_loc}/bin"));
+                install_phase.add_path(format!("{VENV_LOCATION}/bin"));
 
                 install_phase.add_cache_directory(PIP_CACHE_DIR.to_string());
                 install_phase.add_cache_directory(PDM_CACHE_DIR.to_string());
 
+                return Ok(Some(install_phase));
+            } else if app.includes_file("uv.lock") {
+                let install_uv = "pip install uv==$NIXPACKS_UV_VERSION".to_string();
+
+                // Here's how we get UV to play well with the pre-existing non-standard venv location:
+                //
+                // 1. Create a venv which allows us to use pip. pip is not installed globally with nixpkgs py
+                // 2. Install uv via pip
+                // 3. UV_PROJECT_ENVIRONMENT is specified elsewhere so `uv sync` installs packages into the same venv
+
+                let mut install_phase = Phase::install(Some(format!(
+                    "{create_env} && {activate_env} && {install_uv} && uv sync --no-dev --frozen"
+                )));
+
+                install_phase.add_path(format!("{VENV_LOCATION}/bin"));
+                install_phase.add_cache_directory(UV_CACHE_DIR.to_string());
+
                 return Ok(Some(install_phase));
             }
+
             let mut install_phase = Phase::install(Some(format!(
                 "{create_env} && {activate_env} && pip install --upgrade build setuptools && pip install ."
             )));
 
             install_phase.add_file_dependency("pyproject.toml".to_string());
-            install_phase.add_path(format!("{env_loc}/bin"));
+            install_phase.add_path(format!("{VENV_LOCATION}/bin"));
 
             install_phase.add_cache_directory(PIP_CACHE_DIR.to_string());
 
@@ -229,7 +279,7 @@ impl PythonProvider {
             let cmd = format!("{create_env} && {activate_env} && {cmd}");
             let mut install_phase = Phase::install(Some(cmd));
 
-            install_phase.add_path(format!("{env_loc}/bin"));
+            install_phase.add_path(format!("{VENV_LOCATION}/bin"));
             install_phase.add_cache_directory(PIP_CACHE_DIR.to_string());
 
             return Ok(Some(install_phase));
@@ -247,6 +297,12 @@ impl PythonProvider {
             ))));
         }
 
+        // the python package is extracted from pyproject.toml, but this can often not be the desired entrypoint
+        // for this reason we prefer main.py to the module heuristic used in the pyproject.toml logic
+        if app.includes_file("main.py") {
+            return Ok(Some(StartPhase::new("python main.py".to_string())));
+        }
+
         if app.includes_file("pyproject.toml") {
             if let OkResult(meta) = PythonProvider::parse_pyproject(app) {
                 if let Some(entry_point) = meta.entry_point {
@@ -257,10 +313,6 @@ impl PythonProvider {
                 }
             }
         }
-        // falls through
-        if app.includes_file("main.py") {
-            return Ok(Some(StartPhase::new("python main.py".to_string())));
-        }
 
         Ok(None)
     }
@@ -329,11 +381,11 @@ impl PythonProvider {
 
             if parts.len() == 3 {
                 // this is the expected result, but will be unexpected to users
-                println!("Patch version detected in .tool-versions, but not supported in nixpkgs.");
+                println!("Patch python version detected in .tool-versions, but not supported in nixpkgs.");
             } else if parts.len() == 2 {
-                println!("Expected a version string in the format x.y.z from .tool-versions");
+                println!("Expected a python version string in the format x.y.z from .tool-versions");
             } else {
-                println!("Could not find a version string in the format x.y.z or x.y from .tool-versions");
+                println!("Could not find a python version string in the format x.y.z or x.y from .tool-versions");
             }
 
             format!("{}.{}", parts[0], parts[1])
@@ -345,12 +397,18 @@ impl PythonProvider {
         Ok(asdf_versions.get("poetry").cloned())
     }
 
+    fn parse_tool_versions_uv_version(file_content: &str) -> Result<Option<String>> {
+        let asdf_versions = parse_tool_versions_content(file_content);
+        Ok(asdf_versions.get("uv").cloned())
+    }
+
     fn default_python_environment_variables() -> EnvironmentVariables {
         let python_variables = vec![
             ("PYTHONFAULTHANDLER", "1"),
             ("PYTHONUNBUFFERED", "1"),
             ("PYTHONHASHSEED", "random"),
             ("PYTHONDONTWRITEBYTECODE", "1"),
+            // TODO I think this would eliminate the need to include the cache version
             ("PIP_NO_CACHE_DIR", "1"),
             ("PIP_DISABLE_PIP_VERSION_CHECK", "1"),
             ("PIP_DEFAULT_TIMEOUT", "100"),
@@ -425,11 +483,13 @@ impl PythonProvider {
                 PYTHON_NIXPKGS_ARCHIVE.into(),
             ));
         }
+
         let matches = matches.unwrap();
         let python_version = (as_default(matches.get(1)), as_default(matches.get(2)));
 
         // Match major and minor versions
         match python_version {
+            ("3", "13") => Ok((Pkg::new("python313"), PYTHON_NIXPKGS_ARCHIVE.into())),
             ("3", "12") => Ok((Pkg::new("python312"), PYTHON_NIXPKGS_ARCHIVE.into())),
             ("3", "11") => Ok((Pkg::new("python311"), PYTHON_NIXPKGS_ARCHIVE.into())),
             ("3", "10") => Ok((Pkg::new("python310"), PYTHON_NIXPKGS_ARCHIVE.into())),

diff --git a/tests/docker_run_tests.rs b/tests/docker_run_tests.rs
@@ -936,7 +936,7 @@ async fn test_python_asdf_poetry() {
     let name = simple_build("./examples/python-asdf-poetry").await.unwrap();
     let output = run_image(&name, None).await;
 
-    assert!(output.contains("3.12.3"), "{}", output);
+    assert!(output.contains("3.12.7"), "{}", output);
     assert!(output.contains("Poetry (version 1.8.2)"), "{}", output);
 }
 
@@ -1032,6 +1032,13 @@ async fn test_python_poetry() {
     assert!(output.contains("Hello from Python-Poetry"));
 }
 
+#[tokio::test]
+async fn test_python_uv() {
+    let name = simple_build("./examples/python-uv").await.unwrap();
+    let output = run_image(&name, None).await;
+    assert!(output.contains("Hello from Python-Uv"));
+}
+
 #[tokio::test]
 async fn test_python_pdm() {
     let name = simple_build("./examples/python-pdm").await.unwrap();

diff --git a/tests/snapshots/generate_plan_tests__python_django.snap b/tests/snapshots/generate_plan_tests__python_django.snap
@@ -1,6 +1,7 @@
 ---
 source: tests/generate_plan_tests.rs
 expression: plan
+snapshot_kind: text
 ---
 {
   "providers": [],
@@ -35,7 +36,7 @@ expression: plan
       "name": "setup",
       "nixPkgs": [
         "python3",
-        "postgresql",
+        "postgresql.dev",
         "gcc"
       ],
       "nixLibs": [

diff --git a/tests/snapshots/generate_plan_tests__python_postgres.snap b/tests/snapshots/generate_plan_tests__python_postgres.snap
@@ -1,6 +1,7 @@
 ---
 source: tests/generate_plan_tests.rs
 expression: plan
+snapshot_kind: text
 ---
 {
   "providers": [],
@@ -35,7 +36,7 @@ expression: plan
       "name": "setup",
       "nixPkgs": [
         "python3",
-        "postgresql",
+        "postgresql.dev",
         "gcc"
       ],
       "nixLibs": [