From 143d570708a06b5ad283a1fa219a47a17f29ceab Mon Sep 17 00:00:00 2001 From: Joe Neeman Date: Mon, 28 Oct 2024 21:59:18 +0700 Subject: [PATCH] it builds... --- Cargo.lock | 187 ++++++-------- Cargo.toml | 3 + core/src/error/mod.rs | 1 - flake.lock | 19 +- flake.nix | 2 +- git/Cargo.toml | 16 ++ git/examples/fetch.rs | 43 +++ git/src/lib.rs | 187 ++++++++++++++ git/tests/main.rs | 122 +++++++++ package/Cargo.toml | 5 +- package/src/index/mod.rs | 19 +- package/src/index/scrape.rs | 1 + package/src/lib.rs | 97 +++++-- package/src/lock.rs | 40 ++- package/src/manifest.rs | 77 +++--- package/src/resolve.rs | 502 +++++++++++------------------------- 16 files changed, 786 insertions(+), 535 deletions(-) create mode 100644 git/Cargo.toml create mode 100644 git/examples/fetch.rs create mode 100644 git/src/lib.rs create mode 100644 git/tests/main.rs diff --git a/Cargo.lock b/Cargo.lock index 8cab6615e7..3a713511d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,6 +114,9 @@ name = "anyhow" version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" +dependencies = [ + "backtrace", +] [[package]] name = "arbitrary" @@ -1326,9 +1329,9 @@ dependencies = [ [[package]] name = "gix" -version = "0.63.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "984c5018adfa7a4536ade67990b3ebc6e11ab57b3d6cd9968de0947ca99b4b06" +checksum = "9048b8d1ae2104f045cb37e5c450fc49d5d8af22609386bfc739c11ba88995eb" dependencies = [ "gix-actor", "gix-archive", @@ -1337,7 +1340,7 @@ dependencies = [ "gix-commitgraph", "gix-config", "gix-credentials", - "gix-date 0.8.7", + "gix-date", "gix-diff", "gix-dir", "gix-discover", @@ -1350,7 +1353,6 @@ dependencies = [ "gix-ignore", "gix-index", "gix-lock", - "gix-macros", "gix-mailmap", "gix-negotiate", "gix-object", @@ -1387,12 +1389,12 @@ dependencies = [ [[package]] name = "gix-actor" -version = "0.31.5" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0e454357e34b833cc3a00b6efbbd3dd4d18b24b9fb0c023876ec2645e8aa3f2" +checksum = "fc19e312cd45c4a66cd003f909163dc2f8e1623e30a0c0c6df3776e89b308665" dependencies = [ "bstr", - "gix-date 0.8.7", + "gix-date", "gix-utils", "itoa", "thiserror", @@ -1401,14 +1403,15 @@ dependencies = [ [[package]] name = "gix-archive" -version = "0.13.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b6bbebdf0223d1d4a69d6027e8b2482daad8eb1a8d3ec97176c7ec58e796d4" +checksum = "9147c08a55c1398b755539e2cdd63ff690ffe4a2e5e5e0780ee6ef2b49b0a60a" dependencies = [ "bstr", - "gix-date 0.8.7", + "gix-date", "gix-object", "gix-worktree-stream", + "jiff", "thiserror", ] @@ -1475,9 +1478,9 @@ dependencies = [ [[package]] name = "gix-config" -version = "0.37.0" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fafe42957e11d98e354a66b6bd70aeea00faf2f62dd11164188224a507c840" +checksum = "78e797487e6ca3552491de1131b4f72202f282fb33f198b1c34406d765b42bb0" dependencies = [ "bstr", "gix-config-value", @@ -1524,18 +1527,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "gix-date" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eed6931f21491ee0aeb922751bd7ec97b4b2fe8fbfedcb678e2a2dce5f3b8c0" -dependencies = [ - "bstr", - "itoa", - "thiserror", - "time", -] - [[package]] name = "gix-date" version = "0.9.1" @@ -1550,9 +1541,9 @@ dependencies = [ [[package]] name = "gix-diff" -version = "0.44.1" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1996d5c8a305b59709467d80617c9fde48d9d75fd1f4179ea970912630886c9d" +checksum = "92c9afd80fff00f8b38b1c1928442feb4cd6d2232a6ed806b6b193151a3d336c" dependencies = [ "bstr", "gix-command", @@ -1570,9 +1561,9 @@ dependencies = [ [[package]] name = "gix-dir" -version = "0.5.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60c99f8c545abd63abe541d20ab6cda347de406c0a3f1c80aadc12d9b0e94974" +checksum = "0ed3a9076661359a1c5a27c12ad6c3ebe2dd96b8b3c0af6488ab7c128b7bdd98" dependencies = [ "bstr", "gix-discover", @@ -1590,9 +1581,9 @@ dependencies = [ [[package]] name = "gix-discover" -version = "0.32.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc27c699b63da66b50d50c00668bc0b7e90c3a382ef302865e891559935f3dbf" +checksum = "0577366b9567376bc26e815fd74451ebd0e6218814e242f8e5b7072c58d956d2" dependencies = [ "bstr", "dunce", @@ -1618,7 +1609,6 @@ dependencies = [ "gix-hash", "gix-trace", "gix-utils", - "jwalk", "libc", "once_cell", "parking_lot", @@ -1630,9 +1620,9 @@ dependencies = [ [[package]] name = "gix-filter" -version = "0.11.3" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6547738da28275f4dff4e9f3a0f28509f53f94dd6bd822733c91cb306bca61a" +checksum = "4121790ae140066e5b953becc72e7496278138d19239be2e63b5067b0843119e" dependencies = [ "bstr", "encoding_rs", @@ -1708,9 +1698,9 @@ dependencies = [ [[package]] name = "gix-index" -version = "0.33.1" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a9a44eb55bd84bb48f8a44980e951968ced21e171b22d115d1cdcef82a7d73f" +checksum = "0cd4203244444017682176e65fd0180be9298e58ed90bd4a8489a357795ed22d" dependencies = [ "bitflags 2.6.0", "bstr", @@ -1745,38 +1735,27 @@ dependencies = [ "thiserror", ] -[[package]] -name = "gix-macros" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "999ce923619f88194171a67fb3e6d613653b8d4d6078b529b15a765da0edcc17" -dependencies = [ - "proc-macro2 1.0.89", - "quote 1.0.37", - "syn 2.0.85", -] - [[package]] name = "gix-mailmap" -version = "0.23.5" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef6daca6edb6a590c7c0533f3f8e75c54663eb56ce08f46f0891db9fc6f09208" +checksum = "d7d522c8ec2501e1a5b2b4cb54e83cb5d9a52471c9d23b3a1e8dadaf063752f7" dependencies = [ "bstr", "gix-actor", - "gix-date 0.8.7", + "gix-date", "thiserror", ] [[package]] name = "gix-negotiate" -version = "0.13.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ec879fb6307bb63519ba89be0024c6f61b4b9d61f1a91fd2ce572d89fe9c224" +checksum = "b4063bf329a191a9e24b6f948a17ccf6698c0380297f5e169cee4f1d2ab9475b" dependencies = [ "bitflags 2.6.0", "gix-commitgraph", - "gix-date 0.8.7", + "gix-date", "gix-hash", "gix-object", "gix-revwalk", @@ -1786,13 +1765,13 @@ dependencies = [ [[package]] name = "gix-object" -version = "0.42.3" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25da2f46b4e7c2fa7b413ce4dffb87f69eaf89c2057e386491f4c55cadbfe386" +checksum = "2f5b801834f1de7640731820c2df6ba88d95480dc4ab166a5882f8ff12b88efa" dependencies = [ "bstr", "gix-actor", - "gix-date 0.8.7", + "gix-date", "gix-features", "gix-hash", "gix-utils", @@ -1805,12 +1784,12 @@ dependencies = [ [[package]] name = "gix-odb" -version = "0.61.1" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20d384fe541d93d8a3bb7d5d5ef210780d6df4f50c4e684ccba32665a5e3bc9b" +checksum = "a3158068701c17df54f0ab2adda527f5a6aca38fd5fd80ceb7e3c0a2717ec747" dependencies = [ "arc-swap", - "gix-date 0.8.7", + "gix-date", "gix-features", "gix-fs", "gix-hash", @@ -1825,9 +1804,9 @@ dependencies = [ [[package]] name = "gix-pack" -version = "0.51.1" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0594491fffe55df94ba1c111a6566b7f56b3f8d2e1efc750e77d572f5f5229" +checksum = "3223aa342eee21e1e0e403cad8ae9caf9edca55ef84c347738d10681676fd954" dependencies = [ "clru", "gix-chunk", @@ -1917,7 +1896,7 @@ checksum = "cc43a1006f01b5efee22a003928c9eb83dde2f52779ded9d4c0732ad93164e3e" dependencies = [ "bstr", "gix-credentials", - "gix-date 0.9.1", + "gix-date", "gix-features", "gix-hash", "gix-transport", @@ -1940,12 +1919,11 @@ dependencies = [ [[package]] name = "gix-ref" -version = "0.44.1" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3394a2997e5bc6b22ebc1e1a87b41eeefbcfcff3dbfa7c4bd73cb0ac8f1f3e2e" +checksum = "ae0d8406ebf9aaa91f55a57f053c5a1ad1a39f60fdf0303142b7be7ea44311e5" dependencies = [ "gix-actor", - "gix-date 0.8.7", "gix-features", "gix-fs", "gix-hash", @@ -1962,9 +1940,9 @@ dependencies = [ [[package]] name = "gix-refspec" -version = "0.23.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6868f8cd2e62555d1f7c78b784bece43ace40dd2a462daf3b588d5416e603f37" +checksum = "ebb005f82341ba67615ffdd9f7742c87787544441c88090878393d0682869ca6" dependencies = [ "bstr", "gix-hash", @@ -1976,12 +1954,12 @@ dependencies = [ [[package]] name = "gix-revision" -version = "0.27.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b13e43c2118c4b0537ddac7d0821ae0dfa90b7b8dbf20c711e153fb749adce" +checksum = "ba4621b219ac0cdb9256883030c3d56a6c64a6deaa829a92da73b9a576825e1e" dependencies = [ "bstr", - "gix-date 0.8.7", + "gix-date", "gix-hash", "gix-hashtable", "gix-object", @@ -1992,12 +1970,12 @@ dependencies = [ [[package]] name = "gix-revwalk" -version = "0.13.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b030ccaab71af141f537e0225f19b9e74f25fefdba0372246b844491cab43e0" +checksum = "b41e72544b93084ee682ef3d5b31b1ba4d8fa27a017482900e5e044d5b1b3984" dependencies = [ "gix-commitgraph", - "gix-date 0.8.7", + "gix-date", "gix-hash", "gix-hashtable", "gix-object", @@ -2019,9 +1997,9 @@ dependencies = [ [[package]] name = "gix-status" -version = "0.10.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4373d989713809554d136f51bc7da565adf45c91aa4d86ef6a79801621bfc8" +checksum = "f70d35ba639f0c16a6e4cca81aa374a05f07b23fa36ee8beb72c100d98b4ffea" dependencies = [ "bstr", "filetime", @@ -2036,14 +2014,15 @@ dependencies = [ "gix-path", "gix-pathspec", "gix-worktree", + "portable-atomic", "thiserror", ] [[package]] name = "gix-submodule" -version = "0.11.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921cd49924ac14b6611b22e5fb7bbba74d8780dc7ad26153304b64d1272460ac" +checksum = "529d0af78cc2f372b3218f15eb1e3d1635a21c8937c12e2dd0b6fc80c2ca874b" dependencies = [ "bstr", "gix-config", @@ -2097,13 +2076,13 @@ dependencies = [ [[package]] name = "gix-traverse" -version = "0.39.2" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e499a18c511e71cf4a20413b743b9f5bcf64b3d9e81e9c3c6cd399eae55a8840" +checksum = "030da39af94e4df35472e9318228f36530989327906f38e27807df305fccb780" dependencies = [ "bitflags 2.6.0", "gix-commitgraph", - "gix-date 0.8.7", + "gix-date", "gix-hash", "gix-hashtable", "gix-object", @@ -2139,9 +2118,9 @@ dependencies = [ [[package]] name = "gix-validate" -version = "0.8.5" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82c27dd34a49b1addf193c92070bcbf3beaf6e10f16a78544de6372e146a0acf" +checksum = "e187b263461bc36cea17650141567753bc6207d036cedd1de6e81a52f277ff68" dependencies = [ "bstr", "thiserror", @@ -2149,9 +2128,9 @@ dependencies = [ [[package]] name = "gix-worktree" -version = "0.34.1" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26f7326ebe0b9172220694ea69d344c536009a9b98fb0f9de092c440f3efe7a6" +checksum = "c312ad76a3f2ba8e865b360d5cb3aa04660971d16dec6dd0ce717938d903149a" dependencies = [ "bstr", "gix-attributes", @@ -2168,9 +2147,9 @@ dependencies = [ [[package]] name = "gix-worktree-state" -version = "0.11.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ed6205b5f51067a485b11843babcf3304c0799e265a06eb0dde7f69cd85cd8" +checksum = "7b05c4b313fa702c0bacd5068dd3e01671da73b938fade97676859fee286de43" dependencies = [ "bstr", "gix-features", @@ -2188,9 +2167,9 @@ dependencies = [ [[package]] name = "gix-worktree-stream" -version = "0.13.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e35d4896249a41856f44571d94d7583b9f3b9cd1a75eaef4f34a4aa2981bed21" +checksum = "68e81b87c1a3ece22a54b682d6fdc37fbb3977132da972cafe5ec07175fddbca" dependencies = [ "gix-attributes", "gix-features", @@ -2605,16 +2584,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "jwalk" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2735847566356cd2179a2a38264839308f7079fa96e6bd5a42d740460e003c56" -dependencies = [ - "crossbeam", - "rayon", -] - [[package]] name = "kstring" version = "2.0.2" @@ -3099,6 +3068,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "nickel-lang-git" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.20", + "gix", + "tempfile", + "thiserror", +] + [[package]] name = "nickel-lang-lsp" version = "1.8.0" @@ -3145,6 +3125,7 @@ dependencies = [ "directories", "gix", "nickel-lang-core", + "nickel-lang-git", "pubgrub", "semver", "serde", @@ -3229,15 +3210,6 @@ dependencies = [ "libc", ] -[[package]] -name = "num_threads" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" -dependencies = [ - "libc", -] - [[package]] name = "object" version = "0.36.5" @@ -3561,6 +3533,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdd14552ad5f5d743a323c10d576f26822a044355d6601f377d813ece46f38fd" dependencies = [ "rustc-hash 1.1.0", + "serde", "thiserror", ] @@ -4604,9 +4577,7 @@ checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", - "libc", "num-conv", - "num_threads", "powerfmt", "serde", "time-core", diff --git a/Cargo.toml b/Cargo.toml index 93a68e374d..00b846fcfd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "core", "cli", + "git", "vector", "lsp/nls", "lsp/lsp-harness", @@ -24,6 +25,7 @@ readme = "README.md" [workspace.dependencies] nickel-lang-core = { version = "0.9.0", path = "./core", default-features = false } +nickel-lang-git = { version = "0.1.0", path = "./git" } nickel-lang-package = { version = "0.1.0", path = "./package" } nickel-lang-vector = { version = "0.1.0", path = "./vector" } nickel-lang-utils = { version = "0.1.0", path = "./utils" } @@ -57,6 +59,7 @@ derive_more = "0.99" directories = "4.0.1" env_logger = "0.10" git-version = "0.3.5" +gix = "0.66.0" indexmap = "1.9.3" indoc = "2" insta = "1.29.0" diff --git a/core/src/error/mod.rs b/core/src/error/mod.rs index 9b76778831..9de8fd31d8 100644 --- a/core/src/error/mod.rs +++ b/core/src/error/mod.rs @@ -4,7 +4,6 @@ //! [codespan](https://crates.io/crates/codespan-reporting) diagnostic from them. use codespan::ByteIndex; -pub use codespan::{FileId, Files}; pub use codespan_reporting::diagnostic::{Diagnostic, Label, LabelStyle}; use codespan_reporting::files::Files as _; diff --git a/flake.lock b/flake.lock index 27d4ab6869..99b4b1d37c 100644 --- a/flake.lock +++ b/flake.lock @@ -170,17 +170,18 @@ }, "nixpkgs": { "locked": { - "lastModified": 1726937504, - "narHash": "sha256-bvGoiQBvponpZh8ClUcmJ6QnsNKw0EMrCQJARK3bI1c=", - "owner": "NixOS", + "lastModified": 1729880355, + "narHash": "sha256-RP+OQ6koQQLX5nw0NmcDrzvGL8HDLnyXt/jHhL1jwjM=", + "owner": "nixos", "repo": "nixpkgs", - "rev": "9357f4f23713673f310988025d9dc261c20e70c6", + "rev": "18536bf04cd71abd345f9579158841376fdd0c5a", "type": "github" }, "original": { - "id": "nixpkgs", + "owner": "nixos", "ref": "nixos-unstable", - "type": "indirect" + "repo": "nixpkgs", + "type": "github" } }, "nixpkgs-23-11": { @@ -271,11 +272,11 @@ ] }, "locked": { - "lastModified": 1727144949, - "narHash": "sha256-uMZMjoCS2nf40TAE1686SJl3OXWfdfM+BDEfRdr+uLc=", + "lastModified": 1730082698, + "narHash": "sha256-xGP95+G2/esys6FpxrunwwfhirfGsFfPKBJ12MLV1Ps=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "2e19799819104b46019d339e78d21c14372d3666", + "rev": "0d594a39c8f08d81246d06a56e1ccfc04782404f", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 33e44132dd..4a7c489d28 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,6 @@ { inputs = { - nixpkgs.url = "nixpkgs/nixos-unstable"; + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; flake-utils.url = "github:numtide/flake-utils"; pre-commit-hooks = { url = "github:cachix/pre-commit-hooks.nix"; diff --git a/git/Cargo.toml b/git/Cargo.toml new file mode 100644 index 0000000000..7f2e774b0b --- /dev/null +++ b/git/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "nickel-lang-git" +description = "Git utility functions for internal use in nickel" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow.workspace = true +gix = { workspace = true, features = ["blocking-network-client"] } +tempfile.workspace = true +thiserror.workspace = true + +[dev-dependencies] +anyhow = { version = "1.0.86", features = ["backtrace"] } +clap = { version = "4.5.16", features = ["derive"] } +gix = { version = "0.66.0", features = ["blocking-http-transport-reqwest-rust-tls"] } diff --git a/git/examples/fetch.rs b/git/examples/fetch.rs new file mode 100644 index 0000000000..5213d9d4bb --- /dev/null +++ b/git/examples/fetch.rs @@ -0,0 +1,43 @@ +use std::path::PathBuf; + +use clap::{ArgGroup, Parser}; +use nickel_lang_git::{Spec, Target}; + +#[derive(Parser)] +#[command(group = ArgGroup::new("target").args(["branch", "tag", "commit"]))] +struct Args { + repository: String, + + directory: PathBuf, + + #[arg(long)] + branch: Option, + + #[arg(long)] + tag: Option, + + #[arg(long)] + commit: Option, +} + +pub fn main() -> anyhow::Result<()> { + let args = Args::parse(); + + let target = if let Some(branch) = args.branch { + Target::Branch(branch) + } else if let Some(tag) = args.tag { + Target::Tag(tag) + } else if let Some(commit) = args.commit { + Target::Commit(commit) + } else { + Target::Default + }; + + let spec = Spec { + url: args.repository.try_into()?, + target, + }; + + nickel_lang_git::fetch(&spec, args.directory)?; + Ok(()) +} diff --git a/git/src/lib.rs b/git/src/lib.rs new file mode 100644 index 0000000000..d969da49a5 --- /dev/null +++ b/git/src/lib.rs @@ -0,0 +1,187 @@ +use anyhow::anyhow; +use gix::{objs::Kind, ObjectId}; +use std::{num::NonZero, path::Path}; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("io error {error} at {path}")] + Io { + error: std::io::Error, + path: std::path::PathBuf, + }, + + #[error("target `{0}` not found")] + TargetNotFound(Target), + + #[error(transparent)] + Internal(#[from] anyhow::Error), +} + +pub type Result = std::result::Result; + +trait IoResultExt { + fn with_path>(self, path: P) -> Result; +} + +impl IoResultExt for Result { + fn with_path>(self, path: P) -> Result { + self.map_err(|error| Error::Io { + error, + path: path.as_ref().to_owned(), + }) + } +} + +trait InternalResultExt { + fn wrap_err(self) -> Result; +} + +impl> InternalResultExt for Result { + fn wrap_err(self) -> Result { + self.map_err(|e| Error::Internal(e.into())) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Spec { + pub url: gix::Url, + pub target: Target, +} + +/// The different kinds of git "thing" that we can target. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Default)] +pub enum Target { + /// By default, we target the remote HEAD. + #[default] + Default, + /// Target the tip of a specific branch. + Branch(String), + /// Target a specific tag. + Tag(String), + /// Target a specific commit. + /// + /// Currently, we only support a full commit: this needs to be a full hex-encoded + /// sha hash. We could try to support prefixes also, but it requires some work + /// and it *appears* to be inherently less efficient because there doesn't seem + /// to be a way to fetch it in one shot. At least, when `cargo` needs to fetch an + /// abbreviated hash it fetches everything and then looks for the hash among the + /// things that it finds. + Commit(String), +} + +impl std::fmt::Display for Target { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Target::Default => write!(f, "HEAD"), + Target::Branch(branch) => write!(f, "refs/heads/{branch}"), + Target::Tag(tag) => write!(f, "refs/tags/{tag}"), + Target::Commit(c) => write!(f, "{}", c), + } + } +} + +impl Target { + fn refspec(&self) -> String { + self.to_string() + } +} + +fn source_object_id(source: &gix::remote::fetch::Source) -> Result { + match source { + gix::remote::fetch::Source::ObjectId(id) => Ok(*id), + gix::remote::fetch::Source::Ref(r) => { + let (_name, id, peeled) = r.unpack(); + + Ok(peeled + .or(id) + .ok_or_else(|| anyhow!("unborn reference"))? + .to_owned()) + } + } +} + +/// Fetches the contents of a git repository into a target directory. +/// +/// The directory will be created if it doesn't exist yet. The data will be +/// fetched from scratch, even if it has already been fetched before. However, +/// we will try to minimize the amount of data to fetch (for example, by doing a +/// shallow fetch). +/// +/// Only the contents of the git repository will be written to the given +/// directory; the git directory itself will be discarded. +pub fn fetch(spec: &Spec, dir: impl AsRef) -> Result { + let dir = dir.as_ref(); + let parent = dir.parent().ok_or(anyhow!("no parent"))?; + + // Fetch the git directory somewhere temporary. + let git_tempdir = tempfile::tempdir().wrap_err()?; + let repo = gix::init(git_tempdir.path()).wrap_err()?; + let refspec = spec.target.refspec(); + + let remote = repo + .remote_at(spec.url.clone()) + .wrap_err()? + .with_fetch_tags(gix::remote::fetch::Tags::None) + .with_refspecs(Some(refspec.as_str()), gix::remote::Direction::Fetch) + .wrap_err()?; + + // This does similar credentials stuff to the git CLI (e.g. it looks for ssh + // keys if it's a fetch over ssh, or it tries to run `askpass` if it needs + // credentials for https). Maybe we want to have explicit credentials + // configuration instead of or in addition to the default? + let connection = remote.connect(gix::remote::Direction::Fetch).wrap_err()?; + let outcome = connection + .prepare_fetch( + &mut gix::progress::Discard, + gix::remote::ref_map::Options::default(), + ) + .wrap_err()? + // For now, we always fetch shallow. Maybe for the index it's more efficient to + // keep a single repo around and update it? But that might be in another method. + .with_shallow(gix::remote::fetch::Shallow::DepthAtRemote( + NonZero::new(1).unwrap(), + )) + .receive(&mut gix::progress::Discard, &gix::interrupt::IS_INTERRUPTED) + .map_err(|e| match e { + gix::remote::fetch::Error::NoMapping { .. } => { + Error::TargetNotFound(spec.target.clone()) + } + _ => Error::Internal(e.into()), + })?; + + if outcome.ref_map.mappings.len() > 1 { + return Err(anyhow!("we only asked for 1 ref; why did we get more?")).wrap_err(); + } + if outcome.ref_map.mappings.is_empty() { + return Err(Error::TargetNotFound(spec.target.clone())); + } + let object_id = source_object_id(&outcome.ref_map.mappings[0].remote)?; + + let object = repo.find_object(object_id).wrap_err()?; + let commit = object.clone().peel_to_kind(Kind::Commit).wrap_err()?; + let tree_id = object.peel_to_tree().wrap_err()?.id(); + let mut index = repo.index_from_tree(&tree_id).wrap_err()?; + + let tree_tempdir = tempfile::tempdir_in(parent).wrap_err()?; + std::fs::create_dir_all(&tree_tempdir).with_path(&tree_tempdir)?; + + gix::worktree::state::checkout( + &mut index, + tree_tempdir.path(), + repo.objects.clone(), + &gix::progress::Discard, + &gix::progress::Discard, + &gix::interrupt::IS_INTERRUPTED, + gix::worktree::state::checkout::Options { + overwrite_existing: true, + ..Default::default() + }, + ) + .wrap_err()?; + index.write(Default::default()).wrap_err()?; + + // Move the checked-out repo to the right place. + std::fs::rename(tree_tempdir.path(), dir).with_path(dir)?; + + Ok(commit.id) +} diff --git a/git/tests/main.rs b/git/tests/main.rs new file mode 100644 index 0000000000..071f2cd663 --- /dev/null +++ b/git/tests/main.rs @@ -0,0 +1,122 @@ +use std::{path::Path, process::Command}; + +use nickel_lang_git::Target; +use tempfile::tempdir; + +fn write_contents(dir: &Path, branch: &str) { + let contents = dir.join("contents.txt"); + + Command::new("git") + .args(["checkout", "-b", branch]) + .current_dir(dir) + .output() + .unwrap(); + std::fs::write(&contents, branch.as_bytes()).unwrap(); + Command::new("git") + .args(["add", contents.as_os_str().to_str().unwrap()]) + .current_dir(dir) + .output() + .unwrap(); + Command::new("git") + .args(["commit", "-m", "foo"]) + .current_dir(dir) + .output() + .unwrap(); +} + +fn check_contents(dir: &Path, target: Target, contents: &str) { + let out_dir = tempdir().unwrap(); + let spec = nickel_lang_git::Spec { + url: dir.try_into().unwrap(), + target, + }; + nickel_lang_git::fetch(&spec, out_dir.path()).unwrap(); + + let contents_path = out_dir.path().join("contents.txt"); + assert_eq!(1, std::fs::read_dir(out_dir.path()).unwrap().count()); + let actual = std::fs::read_to_string(contents_path).unwrap(); + assert_eq!(contents, actual); +} + +fn check_missing(dir: &Path, target: Target) { + let out_dir = tempdir().unwrap(); + let spec = nickel_lang_git::Spec { + url: dir.try_into().unwrap(), + target, + }; + let result = nickel_lang_git::fetch(&spec, out_dir.path()); + + assert!(matches!( + result, + Err(nickel_lang_git::Error::TargetNotFound(_)) + )); +} + +// Test that the various targets (branch, tag, commit) fetch the expected contents. +#[test] +fn fetch_targets() { + let repo = tempdir().unwrap(); + + Command::new("git") + .arg("init") + .current_dir(repo.path()) + .output() + .unwrap(); + + Command::new("git") + .args(["branch", "-m", "main"]) + .current_dir(repo.path()) + .output() + .unwrap(); + + write_contents(repo.path(), "main"); + write_contents(repo.path(), "other_branch"); + + Command::new("git") + .args(["tag", "a_tag"]) + .current_dir(repo.path()) + .output() + .unwrap(); + let main_hash = Command::new("git") + .args(["rev-parse", "main"]) + .current_dir(repo.path()) + .output() + .unwrap() + .stdout; + let main_hash = std::str::from_utf8(main_hash.trim_ascii()).unwrap(); + let branch_hash = Command::new("git") + .args(["rev-parse", "other_branch"]) + .current_dir(repo.path()) + .output() + .unwrap() + .stdout; + let branch_hash = std::str::from_utf8(branch_hash.trim_ascii()).unwrap(); + // TODO: what do we need to do to support short hashes? + //let short_branch_hash = &branch_hash[0..8]; + + check_contents(repo.path(), Target::Default, "other_branch"); + check_contents(repo.path(), Target::Branch("main".to_owned()), "main"); + check_contents( + repo.path(), + Target::Branch("other_branch".to_owned()), + "other_branch", + ); + check_contents(repo.path(), Target::Commit(main_hash.to_owned()), "main"); + check_contents( + repo.path(), + Target::Commit(branch_hash.to_owned()), + "other_branch", + ); + // check_contents( + // repo.path(), + // Target::Commit(short_branch_hash.to_owned()), + // "other_branch", + // ); + check_contents(repo.path(), Target::Tag("a_tag".to_owned()), "other_branch"); + + check_missing(repo.path(), Target::Tag("not_a_tag".to_owned())); + check_missing(repo.path(), Target::Branch("not_a_branch".to_owned())); + // We don't currently get a nice error for incorrect commit refs, because + // gix doesn't give us back a useful structured error: we get a + // FetchResponse(UploadPack(..)) with a string error message. +} diff --git a/package/Cargo.toml b/package/Cargo.toml index c5e298b511..5dda19d25e 100644 --- a/package/Cargo.toml +++ b/package/Cargo.toml @@ -16,9 +16,10 @@ readme.workspace = true [dependencies] anyhow.workspace = true directories.workspace = true -gix = { version = "0.63.0", features = ["blocking-network-client", "blocking-http-transport-reqwest-rust-tls"] } +gix.workspace = true nickel-lang-core = { workspace = true, default-features = false } -pubgrub = "0.2.1" +nickel-lang-git.workspace = true +pubgrub = { version = "0.2.1", features = ["serde"] } semver = { version = "1.0.23", features = ["serde"] } serde.workspace = true serde_json.workspace = true diff --git a/package/src/index/mod.rs b/package/src/index/mod.rs index 4079a79ef4..2698d7ef7b 100644 --- a/package/src/index/mod.rs +++ b/package/src/index/mod.rs @@ -21,8 +21,8 @@ use serde::{Deserialize, Serialize}; use tempfile::{tempdir_in, NamedTempFile}; use crate::{ - util::{self, cache_dir, clone_git, semver_to_pg}, - Precise, + util::{self, cache_dir, clone_git}, + Precise, VersionReq, }; pub const INDEX_URL: &str = "https://github.com/tweag/nickel-mine.git"; @@ -224,9 +224,9 @@ impl PackageIndex { self.cache.borrow_mut().save(pkg) } - pub fn ensure_downloaded(&self, id: &Id, v: semver::Version) -> anyhow::Result<()> { + pub fn ensure_downloaded(&self, id: &Id, v: SemanticVersion) -> anyhow::Result<()> { let package = self - .package(id, semver_to_pg(v.clone())) + .package(id, v.clone()) .ok_or(anyhow!("tried to download an unknown package"))?; let precise = Precise::Index { id: id.clone(), @@ -277,16 +277,17 @@ impl PackageIndex { } } -/// Packages in the index are identified by an organization and a package name. +/// Packages in the index are identified by a repo (like "github"), an organization, and a package name. #[derive(Clone, PartialEq, Eq, Debug, Hash, Serialize, Deserialize, PartialOrd, Ord)] pub struct Id { + pub repo: String, pub org: String, pub name: String, } impl std::fmt::Display for Id { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}/{}", self.org, self.name) + write!(f, "{}/{}/{}", self.repo, self.org, self.name) } } @@ -294,9 +295,11 @@ impl std::str::FromStr for Id { type Err = (); fn from_str(s: &str) -> Result { - let (org, name) = s.split_once('/').ok_or(())?; + let (repo, org_name) = s.split_once('/').ok_or(())?; + let (org, name) = org_name.split_once('/').ok_or(())?; // TODO: decide on valid identifiers Ok(Id { + repo: repo.to_owned(), org: org.to_owned(), name: name.to_owned(), }) @@ -334,5 +337,5 @@ pub enum PackageLocation { pub struct IndexDependency { #[serde(flatten)] pub id: Id, - pub req: semver::VersionReq, + pub req: VersionReq, } diff --git a/package/src/index/scrape.rs b/package/src/index/scrape.rs index 530e759fdf..f40a913447 100644 --- a/package/src/index/scrape.rs +++ b/package/src/index/scrape.rs @@ -51,6 +51,7 @@ pub fn scrape(id: &Id) -> anyhow::Result { } let package_id = Id { + repo: id.repo.clone(), org: id.org.clone(), name: manifest.name.label().to_owned(), }; diff --git a/package/src/lib.rs b/package/src/lib.rs index d1f53c4a92..c0469bf868 100644 --- a/package/src/lib.rs +++ b/package/src/lib.rs @@ -9,34 +9,77 @@ pub mod util; pub use manifest::ManifestFile; use nickel_lang_core::{cache::normalize_abs_path, package::ObjectId}; -use semver::{Version, VersionReq}; +use pubgrub::version::SemanticVersion; +use semver::Version; use serde::{Deserialize, Serialize}; use util::cache_dir; +// TODO: allow targeting branches or revisions, and allow supplying a relative path +#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize)] +pub struct GitDependency { + #[serde(with = "serde_url")] + url: gix::Url, + #[serde(with = "git_target")] + spec: nickel_lang_git::Target, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize)] +pub enum VersionReq { + Compatible(SemanticVersion), + // TODO: This one could allow pre-releases + Exact(SemanticVersion), +} + +fn next_incompatible(v: &SemanticVersion) -> SemanticVersion { + let (major, _minor, _patch) = (*v).into(); + if major == 0 { + v.bump_minor() + } else { + v.bump_major() + } +} + +impl VersionReq { + pub fn matches(&self, v: &SemanticVersion) -> bool { + match self { + VersionReq::Compatible(lower_bound) => { + lower_bound <= v && *v < next_incompatible(lower_bound) + } + VersionReq::Exact(w) => v == w, + } + } +} + /// A source includes the place to fetch a package from (e.g. git or a registry), /// along with possibly some narrowing-down of the allowed versions (e.g. a range /// of versions, or a git commit id). #[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize)] pub enum Dependency { - // TODO: allow targeting branches or revisions, and allow supplying a relative path - Git { - #[serde(with = "serde_url")] - url: gix::Url, - //tree: Option, - }, - Path { - path: PathBuf, - }, - Index { - id: index::Id, - version: VersionReq, - }, + Git(GitDependency), + Path { path: PathBuf }, + Index { id: index::Id, version: VersionReq }, +} + +/// The same as [`Dependency`], but only for the packages that have fixed, unresolvable, versions. +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub enum UnversionedPackage { + Git(GitDependency), + Path { path: PathBuf }, +} + +impl From for Dependency { + fn from(p: UnversionedPackage) -> Self { + match p { + UnversionedPackage::Git(git) => Dependency::Git(git), + UnversionedPackage::Path { path } => Dependency::Path { path }, + } + } } impl Dependency { pub fn matches(&self, precise: &Precise) -> bool { match (self, precise) { - (Dependency::Git { url }, Precise::Git { repo, .. }) => url == repo, + (Dependency::Git(git), Precise::Git { url: repo, .. }) => &git.url == repo, (Dependency::Path { path }, Precise::Path { path: locked_path }) => path == locked_path, ( Dependency::Index { @@ -44,7 +87,7 @@ impl Dependency { version: dep_version, }, Precise::Index { id, version }, - ) => id == dep_id && dep_version.matches(version), + ) => id == dep_id && dep_version.matches(&version), _ => false, } } @@ -67,10 +110,23 @@ mod serde_url { } } +mod git_target { + use nickel_lang_git::Target; + use serde::{de::Error, Deserialize, Serialize as _}; + + pub fn serialize(url: &Target, ser: S) -> Result { + todo!() + } + + pub fn deserialize<'de, D: serde::Deserializer<'de>>(de: D) -> Result { + todo!() + } +} + #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] pub struct IndexPrecise { id: index::Id, - version: Version, + version: SemanticVersion, } /// A precise package version, in a format suitable for putting into a lockfile. @@ -83,7 +139,7 @@ pub enum Precise { // it, because it allows us to fetch the package if it isn't available, and it allows us to // check if the locked dependency matches the manifest (which might only have the url). #[serde(with = "serde_url")] - repo: gix::Url, + url: gix::Url, id: ObjectId, path: PathBuf, }, @@ -91,11 +147,12 @@ pub enum Precise { /// to the top-level package manifest. /// /// Note that when normalizing we only look at the path and not at the actual filesystem. + /// TODO: maybe just leave out the path altogether? cargo does... Path { path: PathBuf }, Index { // TODO: IndexPrecise id: index::Id, - version: Version, + version: SemanticVersion, }, } @@ -139,7 +196,7 @@ impl Precise { } } - pub fn version(&self) -> Option { + pub fn version(&self) -> Option { match self { Precise::Index { version, .. } => Some(version.clone()), _ => None, diff --git a/package/src/lock.rs b/package/src/lock.rs index 288a93064d..ad435263e6 100644 --- a/package/src/lock.rs +++ b/package/src/lock.rs @@ -63,7 +63,45 @@ pub struct LockFile { impl LockFile { // TODO: move the implementation here pub fn new(manifest: &ManifestFile, resolution: &Resolution) -> Result { - resolution.lock_file(manifest) + // We don't put all packages in the lock file: we ignore dependencies (and therefore also + // transitive dependencies) of path deps. In order to figure out what to include, we + // traverse the depencency graph. + fn collect_packages( + res: &Resolution, + pkg: &Precise, + acc: &mut HashMap, + ) { + let entry = LockFileEntry { + dependencies: if pkg.is_path() { + // Skip dependencies of path deps + Default::default() + } else { + res.dependencies(pkg) + }, + }; + + // Only recurse if this is the first time we've encountered this precise package. + if acc.insert(pkg.clone(), entry).is_none() { + for (_, dep) in acc[pkg].clone().dependencies { + collect_packages(res, &dep, acc); + } + } + } + + let mut acc = HashMap::new(); + for dep in manifest.dependencies.values() { + collect_packages(resolution, &resolution.precise(dep), &mut acc); + } + + Ok(LockFile { + dependencies: manifest + .dependencies + .iter() + .map(|(name, dep)| (*name, resolution.precise(dep))) + .collect(), + + packages: acc, + }) } } diff --git a/package/src/manifest.rs b/package/src/manifest.rs index ecaa448928..248241e2aa 100644 --- a/package/src/manifest.rs +++ b/package/src/manifest.rs @@ -18,9 +18,9 @@ use crate::{ error::{Error, IoResultExt}, lock::LockFile, repo_root, - resolve::Resolution, + resolve::{Resolution, UnversionedPrecise}, util::clone_git, - Dependency, Precise, + Dependency, GitDependency, Precise, }; #[derive(Clone, Debug, Deserialize)] @@ -135,7 +135,7 @@ impl ManifestFile { /// Regenerate the lock file, even if it already exists. pub fn regenerate_lock(&self) -> Result { let resolution = self.resolve()?; - let lock = resolution.lock_file(self)?; + let lock = LockFile::new(self, &resolution)?; if let Some(lock_path) = self.lockfile_path() { // unwrap: serde_json serialization fails if the derived `Serialize` @@ -172,18 +172,11 @@ impl ManifestFile { } } -#[derive(Clone, Debug)] -pub struct RealizedDependency { - /// Either `Git` or `Path`. - pub precise: Precise, - pub manifest: ManifestFile, -} - #[derive(Clone, Debug, Default)] pub struct Realization { - // TODO: the key here should be whatever's in Dependency::Git. Currently that's just the repo url, but it will change - pub git: HashMap, - pub precise: HashMap, + pub git: HashMap, + /// A map from (parent package, dependency) to child package. + pub dependency: HashMap<(Precise, Dependency), UnversionedPrecise>, pub manifests: HashMap, } @@ -191,29 +184,29 @@ impl Realization { // TODO: take in an import sequence (like: the dependency was imported from x, which was imported from y) and use it to improve error messages pub fn realize_all( &mut self, - root_path: Option<&Path>, + root_path: &Path, dep: &Dependency, relative_to: Option<&Precise>, ) -> Result<(), Error> { - let precise = match (dep, relative_to) { + let uprecise = match (dep, relative_to) { // Repo dependencies are resolved later. They are not allowed to have // transitive git or path dependencies, so we don't even need to recurse. (Dependency::Index { .. }, _) => { return Ok(()); } - (Dependency::Git { url }, _) => { - let id = self.realize_one(url)?; - Precise::Git { + (Dependency::Git(git), _) => { + let id = self.realize_one(&git)?; + UnversionedPrecise::Git { id, - repo: url.clone(), + url: git.url.clone(), path: PathBuf::new(), } } - (Dependency::Path { path }, None) => Precise::Path { path: path.clone() }, + (Dependency::Path { path }, None) => UnversionedPrecise::Path { path: path.clone() }, (Dependency::Path { path }, Some(relative_to)) => { let p = normalize_rel_path(&relative_to.local_path().join(path)); match relative_to { - Precise::Git { id, repo, .. } => { + Precise::Git { id, url: repo, .. } => { let repo_path = repo_root(id); let p = p .strip_prefix(&repo_path) @@ -222,40 +215,44 @@ impl Realization { attempted: p.clone(), restriction: repo_path.to_owned(), })?; - Precise::Git { + UnversionedPrecise::Git { id: *id, - repo: repo.clone(), + url: repo.clone(), path: p.to_owned(), } } - _ => Precise::Path { path: p }, + _ => UnversionedPrecise::Path { path: p }, } } }; + let precise = Precise::from(uprecise.clone()); let path = precise.local_path(); - let abs_path = if path.is_absolute() { - path - } else if let Some(root_path) = root_path { - root_path.join(path) - } else { - return Err(Error::NoPackageRoot { path }); - }; - let manifest = ManifestFile::from_path(abs_path.join("package.ncl"))?; - self.precise.insert(dep.clone(), precise.clone()); - self.manifests.insert(precise.clone(), manifest.clone()); + let abs_path = root_path.join(path); - for dep in manifest.dependencies.values() { - self.realize_all(root_path, dep, Some(&precise))?; + if !self.manifests.contains_key(&precise) { + let manifest = ManifestFile::from_path(abs_path.join("package.ncl"))?; + + let parent_precise = relative_to.cloned().unwrap_or_else(|| Precise::Path { + path: root_path.to_owned(), + }); + self.dependency + .insert((parent_precise, dep.clone()), uprecise); + self.manifests.insert(precise.clone(), manifest.clone()); + + for dep in manifest.dependencies.values() { + self.realize_all(root_path, dep, Some(&precise))?; + } } Ok(()) } - fn realize_one(&mut self, url: &gix::Url) -> Result { - if let Some(id) = self.git.get(url) { + fn realize_one(&mut self, git: &GitDependency) -> Result { + if let Some(id) = self.git.get(git) { return Ok(*id); } + let url = &git.url; fn err(url: &gix::Url, msg: impl std::fmt::Display) -> Error { Error::Git { @@ -280,7 +277,7 @@ impl Realization { // Now that we know the object hash, move the fetched repo to the right place in the cache. let precise = Precise::Git { id, - repo: url.clone(), + url: url.clone(), path: PathBuf::default(), }; let path = precise.local_path(); @@ -292,7 +289,7 @@ impl Realization { std::fs::rename(tmp_dir, &path).with_path(path)?; } - self.git.insert(url.clone(), id); + self.git.insert(git.clone(), id); Ok(id) } } diff --git a/package/src/resolve.rs b/package/src/resolve.rs index e428724aa4..529b4d24ac 100644 --- a/package/src/resolve.rs +++ b/package/src/resolve.rs @@ -2,20 +2,13 @@ //! copies of a package, but we insist that all semver-compatible verisons must resolve //! to the exact same version. //! -//! This is not natively supported in pubgrub, so we use the two transformations described -//! in [their book](https://pubgrub-rs-guide.pages.dev/limitations/multiple_versions). -//! The first transformation is to make a new package for every collection of semver-compatible +//! This is not natively supported in pubgrub, so we use one of the two transformations described +//! in [their book](https://pubgrub-rs-guide.pages.dev/limitations/multiple_versions): +//! we make a new package for every collection of semver-compatible //! versions of each package. So instead of having `foo` with versions `1.1`, `1.2` and `2.0`, //! we have a package `foo#1` with versions `1.1` and `1.2` and another package `foo#2` -//! with version `2.0`. We call `foo#1` and `foo#2` "buckets". Since we present them to pubgrub +//! with version `2.0`. Since we present them to pubgrub //! as different packages, they can both appear in the final resolution. -//! -//! The problem with the approach above is that it introduces "union" dependencies, where -//! a package that required `foo` at version `>=1.0` now has a dependency on either `foo#1` -//! or `foo#2`. Pubgrub doesn't natively support union dependencies, so there's another -//! transformation needed for that. Any time there's a dependency that depends on one of -//! multiple buckets, we insert a "union" package (pubgrub's write-up calls it a "proxy") -//! that has a version for each bucket. See the pubgrub write-up for more details. use std::{ borrow::Borrow, @@ -29,15 +22,16 @@ use pubgrub::{ solver::DependencyProvider, version::{SemanticVersion, Version}, }; -use semver::{Comparator, VersionReq}; +use semver::Comparator; use crate::{ error::{Error, IoResultExt as _}, index::{Id, IndexDependency, PackageIndex}, - lock::{LockFile, LockFileEntry}, + lock::LockFile, manifest::Realization, util::semver_to_pg, - Dependency, IndexPrecise, ManifestFile, Precise, + Dependency, GitDependency, IndexPrecise, ManifestFile, ObjectId, Precise, UnversionedPackage, + VersionReq, }; type VersionRange = pubgrub::range::Range; @@ -47,7 +41,7 @@ pub struct PackageRegistry { // those same versions. We won't absolutely insist on it, because if the manifest // changed (or some path-dependency changed) then the old locked versions might not // resolve anymore. - previously_locked: HashMap, + previously_locked: HashMap, index: PackageIndex, realized_unversioned: Realization, } @@ -55,14 +49,14 @@ pub struct PackageRegistry { impl PackageRegistry { pub fn list_versions<'a>( &'a self, - package: &VirtualPackage, + package: &Package, ) -> impl Iterator + 'a { let locked_version = self.previously_locked.get(package).cloned(); let rest = match package { - VirtualPackage::Package(Package::Unversioned(_)) => { + Package::Unversioned(_) => { Box::new(std::iter::once(SemanticVersion::zero())) as Box> } - VirtualPackage::Package(Package::Bucket(b)) => { + Package::Bucket(b) => { let bucket_version = b.version; let iter = self .index @@ -70,24 +64,6 @@ impl PackageRegistry { .filter(move |v| bucket_version.contains(*v)); Box::new(iter) } - // The edge package has a version for every bucket of target versions that - // it could potentially resolve to. - VirtualPackage::Union { - source, - source_version, - target, - } => { - // First, find all buckets of target versions. - let target_versions = bucket_versions(self.index.available_versions(target)); - // Filter to keep only the ones that could concievably intersect with - // the allowed version constraints. - let source_req = self.dep(source, source_version, target); - - let iter = target_versions - .filter(move |v| BucketVersion::from(*v).intersects(&source_req)); - - Box::new(iter) - } }; // Put the locked version first, and then the other versions in any order (filtering to ensure that the locked version isn't repeated). @@ -109,10 +85,9 @@ impl PackageRegistry { .unwrap() } - pub fn unversioned_deps(&self, pkg: &UnversionedPackage) -> Vec { - let dep = Dependency::from(pkg.clone()); - let precise = &self.realized_unversioned.precise[&dep]; - let manifest = &self.realized_unversioned.manifests[precise]; + pub fn unversioned_deps(&self, pkg: &UnversionedPrecise) -> Vec { + let precise = Precise::from(pkg.clone()); + let manifest = &self.realized_unversioned.manifests[&precise]; manifest.dependencies.values().cloned().collect() } @@ -141,22 +116,6 @@ fn bucket_versions( vs.into_iter() } -/// The same as [`Dependency`], but only for the packages that have fixed, unresolvable, versions. -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub enum UnversionedPackage { - Git { url: gix::Url }, - Path { path: PathBuf }, -} - -impl From for Dependency { - fn from(p: UnversionedPackage) -> Self { - match p { - UnversionedPackage::Git { url } => Dependency::Git { url }, - UnversionedPackage::Path { path } => Dependency::Path { path }, - } - } -} - /// A bucket version represents a collection of compatible semver versions. #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] pub enum BucketVersion { @@ -193,36 +152,6 @@ impl BucketVersion { } } - pub fn intersects(&self, req: &VersionReq) -> bool { - req.comparators - .iter() - .all(|c| self.intersects_constraint(c)) - } - - pub fn intersects_constraint(&self, comp: &Comparator) -> bool { - let maj = comp.major as u32; // TODO: avoid casts - let min = comp.minor.unwrap_or_default() as u32; - let comp_version = SemanticVersion::new(maj, min, 0); - - let slf: SemanticVersion = (*self).into(); - let next: SemanticVersion = self.next().into(); - let range = slf..next; - match comp.op { - semver::Op::Exact => range.contains(&comp_version), - semver::Op::Greater => next > comp_version, - semver::Op::GreaterEq => next > comp_version, - semver::Op::Less => slf < comp_version, - semver::Op::LessEq => slf <= comp_version, - semver::Op::Tilde => range.contains(&comp_version), - semver::Op::Caret => range.contains(&comp_version), - semver::Op::Wildcard => true, - // This is silly. Semver insists on having op be non_exhaustive (https://github.com/dtolnay/semver/issues/262) - // even though the addition of new ops should (IMO) be semver-breaking. We don't - // expect any other ops, but we have to handle them somehow. - _ => false, - } - } - pub fn compatible_range(&self) -> VersionRange { VersionRange::between( SemanticVersion::from(*self), @@ -251,28 +180,58 @@ impl From for SemanticVersion { } } +impl From for BucketVersion { + fn from(v: VersionReq) -> Self { + match v { + VersionReq::Compatible(v) | VersionReq::Exact(v) => v.into(), + } + } +} + #[derive(Debug, Clone, Eq, PartialEq, Hash)] pub struct Bucket { pub id: Id, pub version: BucketVersion, } +/// Identical to `Precise`, but contains only the unversioned variants. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum UnversionedPrecise { + Git { + url: gix::Url, + id: ObjectId, + path: PathBuf, + }, + Path { + path: PathBuf, + }, +} + +impl From for Precise { + fn from(up: UnversionedPrecise) -> Self { + match up { + UnversionedPrecise::Git { url, id, path } => Precise::Git { url, id, path }, + UnversionedPrecise::Path { path } => Precise::Path { path }, + } + } +} + #[derive(Debug, Clone, Eq, PartialEq, Hash)] pub enum Package { /// A package that only comes in one version (like a path or a git dependency). /// TODO: right now we say that all unversioned packages have version `0.0.0`, but it /// isn't great for error messages - Unversioned(UnversionedPackage), + Unversioned(UnversionedPrecise), Bucket(Bucket), } impl std::fmt::Display for Package { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Package::Unversioned(UnversionedPackage::Git { url }) => { - write!(f, "{url}") + Package::Unversioned(UnversionedPrecise::Git { url, .. }) => { + write!(f, "{}", url) } - Package::Unversioned(UnversionedPackage::Path { path }) => { + Package::Unversioned(UnversionedPrecise::Path { path }) => { write!(f, "{}", path.display()) } Package::Bucket(b) => { @@ -286,62 +245,21 @@ impl std::fmt::Display for Package { impl From for Package { fn from(p: Precise) -> Self { match p { - Precise::Git { repo, .. } => { - Package::Unversioned(UnversionedPackage::Git { url: repo }) + Precise::Git { url, id, path } => { + Package::Unversioned(UnversionedPrecise::Git { url, id, path }) } - Precise::Path { path } => Package::Unversioned(UnversionedPackage::Path { path }), + Precise::Path { path } => Package::Unversioned(UnversionedPrecise::Path { path }), Precise::Index { id, version } => Package::Bucket(Bucket { id, - version: semver_to_pg(version).into(), + version: version.into(), }), } } } -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub enum VirtualPackage { - Package(Package), - Union { - source: Package, - source_version: SemanticVersion, - target: Id, - }, -} - -fn proxify_dep(pkg: &Package, pkg_version: SemanticVersion, dep: Dependency) -> VirtualPackage { - match dep { - Dependency::Git { url } => { - VirtualPackage::Package(Package::Unversioned(UnversionedPackage::Git { url })) - } - Dependency::Path { path } => { - VirtualPackage::Package(Package::Unversioned(UnversionedPackage::Path { path })) - } - // We're making a proxy for every dependency on a repo package, but we could skip - // the proxy if the dependency range stays within a single semver range. - Dependency::Index { id, .. } => VirtualPackage::Union { - source: pkg.clone(), - source_version: pkg_version, - target: id, - }, - } -} - -impl std::fmt::Display for VirtualPackage { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - VirtualPackage::Package(b) => b.fmt(f), - VirtualPackage::Union { - source, - source_version, - target, - } => write!(f, "{}@{}->{}", source, source_version, target), - } - } -} - -impl DependencyProvider for PackageRegistry { +impl DependencyProvider for PackageRegistry { fn choose_package_version< - T: Borrow, + T: Borrow, U: Borrow>, >( &self, @@ -370,103 +288,86 @@ impl DependencyProvider for PackageRegistry { fn get_dependencies( &self, - package: &VirtualPackage, + package: &Package, version: &SemanticVersion, - ) -> Result< - pubgrub::solver::Dependencies, - Box, - > { + ) -> Result, Box> + { match package { - VirtualPackage::Package(b @ Package::Unversioned(p)) => { + Package::Unversioned(p) => { + let precise = Precise::from(p.clone()); let deps = self .unversioned_deps(p) .into_iter() - .map(|dep| (proxify_dep(b, *version, dep), VersionRange::any())) + .map(|dep| { + match dep { + Dependency::Git(_) | Dependency::Path { .. } => { + let dep_precise = self.realized_unversioned.dependency + [&(precise.clone(), dep.clone())] + .clone(); + ( + Package::Unversioned(dep_precise.into()), + VersionRange::any(), + ) + } + // We're making a proxy for every dependency on a repo package, but we could skip + // the proxy if the dependency range stays within a single semver range. + Dependency::Index { id, version } => { + let pkg = Package::Bucket(Bucket { + id, + version: version.clone().into(), + }); + + let range = match version { + VersionReq::Compatible(v) => VersionRange::higher_than(v), + VersionReq::Exact(v) => VersionRange::exact(v), + }; + + (pkg, range) + } + } + }) .collect(); Ok(pubgrub::solver::Dependencies::Known(deps)) } - VirtualPackage::Package(Package::Bucket(b)) => { + Package::Bucket(b) => { let deps = self .index_deps(&b.id, version) .into_iter() .map(|dep| { - ( - proxify_dep(&Package::Bucket(b.clone()), *version, dep), - VersionRange::any(), - ) + let Dependency::Index { id, version } = dep else { + panic!("index packages can only have index deps"); + }; + + // FIXME: copy-paste from the other branch + let pkg = Package::Bucket(Bucket { + id, + version: version.clone().into(), + }); + + let range = match version { + VersionReq::Compatible(v) => VersionRange::higher_than(v), + VersionReq::Exact(v) => VersionRange::exact(v), + }; + + (pkg, range) }) .collect(); Ok(pubgrub::solver::Dependencies::Known(deps)) } - VirtualPackage::Union { - source, - source_version, - target, - } => { - // A proxy package depends on a single package: its target bucket with the - // same version as itself. - let bucket_version = (*version).into(); - let dep = Bucket { - id: target.clone(), - version: bucket_version, - }; - let range = version_req_to_range(&self.dep(source, source_version, target)); - let bucket_range = bucket_version.compatible_range().intersection(&range); - let deps: HashMap<_, _, _> = - std::iter::once((VirtualPackage::Package(Package::Bucket(dep)), bucket_range)) - .collect(); - Ok(pubgrub::solver::Dependencies::Known(deps)) - } } } } fn version_req_to_range(req: &VersionReq) -> pubgrub::range::Range { - use pubgrub::range::Range; - - let mut ret = pubgrub::range::Range::any(); - - fn comp_to_range(comp: &Comparator) -> pubgrub::range::Range { - let v = SemanticVersion::new( - comp.major as u32, - comp.minor.unwrap_or_default() as u32, - comp.patch.unwrap_or_default() as u32, - ); - match comp.op { - semver::Op::Exact => Range::exact(v), - semver::Op::Greater => Range::higher_than(v.bump()), - semver::Op::GreaterEq => Range::higher_than(v), - semver::Op::Less => Range::strictly_lower_than(v), - semver::Op::LessEq => Range::strictly_lower_than(v).union(&Range::exact(v)), - semver::Op::Tilde => { - if comp.minor.is_some() { - Range::between(v, v.bump_minor()) - } else { - Range::between(v, v.bump_major()) - } - } - semver::Op::Caret => { - if comp.major == 0 { - Range::between(v, v.bump_minor()) - } else { - Range::between(v, v.bump_major()) - } - } - semver::Op::Wildcard => Range::any(), - _ => panic!("unknown op"), - } + match req { + VersionReq::Compatible(v) => VersionRange::higher_than(v.clone()), + VersionReq::Exact(v) => VersionRange::exact(v.clone()), } - - for comp in &req.comparators { - ret = ret.intersection(&comp_to_range(comp)); - } - - ret } pub struct Resolution { pub realization: Realization, - pub package_map: HashMap>, + pub package_map: HashMap>, pub index: PackageIndex, } @@ -474,10 +375,7 @@ pub fn resolve(manifest: &ManifestFile) -> Result { resolve_with_lock(manifest, &LockFile::default()) } -fn previously_locked( - top_level: &Package, - lock: &LockFile, -) -> HashMap { +fn previously_locked(top_level: &Package, lock: &LockFile) -> HashMap { fn precise_to_index(p: &Precise) -> Option { match p { Precise::Index { id, version } => Some(IndexPrecise { @@ -493,45 +391,22 @@ fn previously_locked( .dependencies .values() .filter_map(precise_to_index) - // FIXME: another place we're hardcoding an unversioned version - .map(|dep| (top_level.clone(), SemanticVersion::zero(), dep)) - .chain(lock.packages.iter().flat_map(|(parent, entry)| { - let pkg = Package::from(parent.clone()); - let pkg_version = parent - .version() - .map(semver_to_pg) - .unwrap_or(SemanticVersion::zero()); - entry - .dependencies + .chain( + lock.packages .values() - .filter_map(precise_to_index) - .map(move |v| (pkg.clone(), pkg_version, v)) - })); + .flat_map(|entry| entry.dependencies.values().filter_map(precise_to_index)), + ); - // Because of the virtual package system, each parent->dep needs two entries - // in the locked map: one for recording which of the "union" packages the parent - // depends on, and another for recording which precise version the "union" package depends on. pkg_deps - .flat_map(|(pkg, version, dep)| { - let dep_version = semver_to_pg(dep.version); - let dep_bucket: BucketVersion = dep_version.into(); - [ - ( - VirtualPackage::Union { - source: pkg, - source_version: version, - target: dep.id.clone(), - }, - dep_bucket.into(), - ), - ( - VirtualPackage::Package(Package::Bucket(Bucket { - id: dep.id, - version: dep_bucket, - })), - dep_version, - ), - ] + .map(|IndexPrecise { id, version }| { + let dep_bucket: BucketVersion = version.into(); + ( + Package::Bucket(Bucket { + id, + version: dep_bucket, + }), + version, + ) }) .collect() } @@ -539,58 +414,42 @@ fn previously_locked( pub fn resolve_with_lock(manifest: &ManifestFile, lock: &LockFile) -> Result { let mut realization = Realization::default(); - // pubgrub insists on resolving a top-level package. We'll represent it as a `Path` dependency, - // so it needs a path... + // FIXME: figure out how to represent the top-level package, recalling that `realization` assumes + // that every time we need to look up a dependency we need a parent package. let root_path = manifest.parent_dir.as_deref(); for dep in manifest.dependencies.values() { - realization.realize_all(root_path, dep, None)?; + // FIXME: unwrap + realization.realize_all(root_path.unwrap(), dep, None)?; } - // The top-level package doesn't matter for resolution, but it might appear in error messages. - // Try to give it an informative name. - let top_level_path = root_path.unwrap_or(Path::new("/dummy-package")); - let top_level_dep = UnversionedPackage::Path { - path: top_level_path.to_path_buf(), - }; - let top_level = VirtualPackage::Package(Package::Unversioned(top_level_dep.clone())); - let precise = Precise::Path { - path: top_level_path.to_path_buf(), + let top_level = UnversionedPrecise::Path { + path: root_path.unwrap().to_path_buf(), }; realization - .precise - .insert(top_level_dep.clone().into(), precise.clone()); - realization.manifests.insert(precise, manifest.clone()); + .manifests + .insert(top_level.clone().into(), manifest.clone()); + let top_level_pkg = Package::Unversioned(top_level); let registry = PackageRegistry { - previously_locked: dbg!(previously_locked( - &Package::Unversioned(top_level_dep), - lock - )), + previously_locked: dbg!(previously_locked(&top_level_pkg, lock)), index: PackageIndex::new(), realized_unversioned: realization, }; registry.index.refresh_from_github(); - let resolution = match pubgrub::solver::resolve(®istry, top_level, SemanticVersion::zero()) { - Ok(r) => r, - Err(pubgrub::error::PubGrubError::NoSolution(derivation_tree)) => { - //derivation_tree.collapse_no_versions(); - let msg = DefaultStringReporter::report(&derivation_tree); - return Err(Error::Resolution { msg }); - } - Err(e) => return Err(Error::Resolution { msg: e.to_string() }), - }; - let mut selected = HashMap::>::new(); - for (virt, vers) in resolution.iter() { - if let VirtualPackage::Package(Package::Bucket(Bucket { id, .. })) = virt { - let (major, minor, patch) = (*vers).into(); - selected - .entry(id.clone()) - .or_default() - .push(semver::Version::new( - major.into(), - minor.into(), - patch.into(), - )); + let resolution = + match pubgrub::solver::resolve(®istry, top_level_pkg, SemanticVersion::zero()) { + Ok(r) => r, + Err(pubgrub::error::PubGrubError::NoSolution(derivation_tree)) => { + //derivation_tree.collapse_no_versions(); + let msg = DefaultStringReporter::report(&derivation_tree); + return Err(Error::Resolution { msg }); + } + Err(e) => return Err(Error::Resolution { msg: e.to_string() }), + }; + let mut selected = HashMap::>::new(); + for (pkg, vers) in resolution.iter() { + if let Package::Bucket(Bucket { id, .. }) = pkg { + selected.entry(id.clone()).or_default().push(vers.clone()); } } Ok(Resolution { @@ -609,9 +468,9 @@ impl Resolution { /// was generated for. pub fn precise(&self, dep: &Dependency) -> Precise { match dep { - Dependency::Git { url } => Precise::Git { - repo: url.clone(), - id: self.realization.git[url], + Dependency::Git(git) => Precise::Git { + url: git.url.clone(), + id: self.realization.git[git], path: PathBuf::new(), }, Dependency::Path { path } => Precise::Path { @@ -641,17 +500,7 @@ impl Resolution { .collect() } Precise::Index { id, version } => { - let pkg = self - .index - .package( - id, - SemanticVersion::new( - version.major as u32, - version.minor as u32, - version.patch as u32, - ), - ) - .unwrap(); + let pkg = self.index.package(id, version.clone()).unwrap(); pkg.deps .into_iter() .map(move |(dep_name, dep)| { @@ -668,7 +517,12 @@ impl Resolution { /// Returns all the resolved packages in the dependency tree. pub fn all_precises(&self) -> Vec { - let mut ret: Vec<_> = self.realization.precise.values().cloned().collect(); + let mut ret: Vec<_> = self + .realization + .dependency + .values() + .map(|p| p.clone().into()) + .collect(); ret.sort(); ret.dedup(); @@ -719,46 +573,4 @@ impl Resolution { .collect(), }) } - - pub fn lock_file(&self, manifest: &ManifestFile) -> Result { - // We don't put all packages in the lock file: we ignore dependencies (and therefore also - // transitive dependencies) of path deps. In order to figure out what to include, we - // traverse the depencency graph. - fn collect_packages( - slf: &Resolution, - pkg: &Precise, - acc: &mut HashMap, - ) { - let entry = LockFileEntry { - dependencies: if pkg.is_path() { - // Skip dependencies of path deps - Default::default() - } else { - slf.dependencies(pkg) - }, - }; - - // Only recurse if this is the first time we've encountered this precise package. - if acc.insert(pkg.clone(), entry).is_none() { - for (_, dep) in acc[pkg].clone().dependencies { - collect_packages(slf, &dep, acc); - } - } - } - - let mut acc = HashMap::new(); - for dep in manifest.dependencies.values() { - collect_packages(self, &self.precise(dep), &mut acc); - } - - Ok(LockFile { - dependencies: manifest - .dependencies - .iter() - .map(|(name, dep)| (*name, self.precise(dep))) - .collect(), - - packages: acc, - }) - } }