diff --git a/Cargo.lock b/Cargo.lock index a8ba47dcc..8d00ddac4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,18 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.2" @@ -26,6 +38,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "anstream" version = "0.6.13" @@ -74,6 +92,27 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -110,6 +149,18 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bitflags" version = "1.3.2" @@ -121,6 +172,22 @@ name = "bitflags" version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +dependencies = [ + "serde", +] + +[[package]] +name = "blake3" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] [[package]] name = "block-buffer" @@ -137,6 +204,12 @@ version = "3.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.5.0" @@ -154,9 +227,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.88" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02f341c093d19155a6e41631ce5971aac4e9a868262212153124c15fa22d1cdc" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" +dependencies = [ + "shlex", +] [[package]] name = "cfg-if" @@ -210,7 +286,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.77", ] [[package]] @@ -271,6 +347,15 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "console" version = "0.15.8" @@ -284,6 +369,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.6.0" @@ -318,6 +415,36 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "crypto-common" version = "0.1.6" @@ -370,7 +497,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.51", + "syn 2.0.77", ] [[package]] @@ -381,7 +508,27 @@ checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core", "quote", - "syn 2.0.51", + "syn 2.0.77", +] + +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", ] [[package]] @@ -391,6 +538,7 @@ dependencies = [ "clap", "cli-table", "console", + "devenv-eval-cache", "dotlock", "fs2", "hex", @@ -408,6 +556,7 @@ dependencies = [ "serde_json", "serde_yaml", "sha2", + "sqlx", "tempdir", "tempfile", "test-log", @@ -419,6 +568,24 @@ dependencies = [ "xdg", ] +[[package]] +name = "devenv-eval-cache" +version = "0.1.0" +dependencies = [ + "blake3", + "futures", + "lazy_static", + "miette", + "regex", + "serde", + "serde_json", + "serde_repr", + "sqlx", + "tempdir", + "thiserror", + "tokio", +] + [[package]] name = "devenv-run-tests" version = "0.1.0" @@ -442,7 +609,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", + "subtle", ] [[package]] @@ -466,6 +635,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dotlock" version = "0.5.0" @@ -486,6 +661,9 @@ name = "either" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +dependencies = [ + "serde", +] [[package]] name = "encode_unicode" @@ -539,6 +717,28 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + [[package]] name = "fastrand" version = "2.0.1" @@ -551,6 +751,17 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -597,6 +808,21 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -604,6 +830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -612,12 +839,45 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "futures-sink" version = "0.3.30" @@ -636,8 +896,11 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", + "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -667,7 +930,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.51", + "syn 2.0.77", ] [[package]] @@ -718,9 +981,22 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown", +] [[package]] name = "heck" @@ -740,6 +1016,24 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.9" @@ -901,9 +1195,12 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "libc" @@ -911,6 +1208,12 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "libredox" version = "0.0.1" @@ -919,7 +1222,18 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ "bitflags 2.4.2", "libc", - "redox_syscall", + "redox_syscall 0.4.1", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", ] [[package]] @@ -928,6 +1242,16 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.22" @@ -952,6 +1276,16 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.1" @@ -986,7 +1320,7 @@ checksum = "dcf09caffaac8068c346b6df2a7fc27a177fd20b39421a39ce0a211bde679a6c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.77", ] [[package]] @@ -995,6 +1329,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.2" @@ -1046,6 +1386,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1056,6 +1406,59 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + [[package]] name = "object" version = "0.32.2" @@ -1094,7 +1497,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.77", ] [[package]] @@ -1133,6 +1536,50 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caff54706df99d2a78a5a4e3455ff45448d81ef1bb63c22cd14052ca0e993a3f" +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.6", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1161,12 +1608,48 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + [[package]] name = "pretty_assertions" version = "1.4.0" @@ -1179,9 +1662,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -1208,6 +1691,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_core" version = "0.3.1" @@ -1223,6 +1727,15 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -1241,6 +1754,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" +dependencies = [ + "bitflags 2.4.2", +] + [[package]] name = "redox_users" version = "0.4.4" @@ -1311,7 +1833,7 @@ version = "0.11.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78bf93c4af7a8bb7d879d51cebe797356ff10ae8516ace542b5182d9dcac10b2" dependencies = [ - "base64", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -1351,6 +1873,26 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b833d8d034ea094b1ea68aa6d5c740e0d04bad9d16568d08ba6f76823a114316" +[[package]] +name = "rsa" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -1376,7 +1918,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.7", ] [[package]] @@ -1457,7 +1999,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.77", ] [[package]] @@ -1466,6 +2008,12 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f3adfbe1c90a6a9643433e490ef1605c6a99f93be37e4c83fe5149fca9698c6" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "2.9.2" @@ -1506,7 +2054,7 @@ checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.77", ] [[package]] @@ -1542,6 +2090,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1567,6 +2126,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.8" @@ -1587,6 +2157,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -1596,6 +2172,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "slab" version = "0.4.9" @@ -1610,6 +2196,9 @@ name = "smallvec" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +dependencies = [ + "serde", +] [[package]] name = "smawk" @@ -1627,6 +2216,233 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlformat" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bba3a93db0cc4f7bdece8bb09e77e2e785c20bfebf79eb8340ed80708048790" +dependencies = [ + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93334716a037193fac19df402f8571269c84a00852f6a7066b5d2616dcd64d3e" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d8060b456358185f7d50c55d9b5066ad956956fddec42ee2e8567134a8936e" +dependencies = [ + "atoi", + "byteorder", + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown", + "hashlink", + "hex", + "indexmap", + "log", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror", + "time", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cac0692bcc9de3b073e8d747391827297e075c7710ff6276d9f7a1f3d58c6657" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.77", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1804e8a7c7865599c9c79be146dc8a9fd8cc86935fa641d3ea58e5f0688abaa5" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.77", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64bb4714269afa44aef2755150a0fc19d756fb580a67db8885608cf02f47d06a" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.4.2", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "time", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fa91a732d854c5d7726349bb4bb879bb9478993ceb764247660aee25f67c2f8" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.4.2", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.5", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "time", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5b2cf34a45953bfd3daaf3db0f7a7878ab9b7a6b91b422d24a7a9e4c857b680" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "time", + "tracing", + "url", +] + [[package]] name = "starbase_styles" version = "0.3.0" @@ -1644,6 +2460,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.10.0" @@ -1656,6 +2483,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "supports-color" version = "3.0.0" @@ -1690,9 +2523,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.51" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab617d94515e94ae53b8406c628598680aa0c9587474ecbe58188f7b345d66c" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -1742,7 +2575,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" dependencies = [ - "rand", + "rand 0.4.6", "remove_dir_all", ] @@ -1797,7 +2630,7 @@ checksum = "5999e24eaa32083191ba4e425deb75cdf25efefabe5aaccb7446dd0d4122a3f5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.77", ] [[package]] @@ -1828,7 +2661,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.77", ] [[package]] @@ -1841,6 +2674,37 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1881,7 +2745,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.77", ] [[package]] @@ -1894,6 +2758,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.10" @@ -1920,6 +2795,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -1933,7 +2809,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.77", ] [[package]] @@ -2019,6 +2895,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524" + [[package]] name = "unicode-segmentation" version = "1.11.0" @@ -2031,6 +2913,12 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "unsafe-libyaml" version = "0.2.10" @@ -2114,7 +3002,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.77", "wasm-bindgen-shared", ] @@ -2148,7 +3036,7 @@ checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.77", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2188,7 +3076,7 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ - "redox_syscall", + "redox_syscall 0.4.1", "wasite", "web-sys", ] @@ -2404,3 +3292,30 @@ name = "yansi" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/Cargo.toml b/Cargo.toml index d96c997aa..a97ade945 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["devenv", "devenv-run-tests", "xtask", "tasks"] +members = ["devenv", "devenv-eval-cache", "devenv-run-tests", "tasks", "xtask"] [workspace.package] edition = "2021" @@ -9,14 +9,23 @@ homepage = "https://devenv.sh/" repository = "https://github.com/cachix/devenv/" [workspace.dependencies] +devenv = { path = "devenv" } +devenv-eval-cache = { path = "devenv-eval-cache" } +devenv-run-tests = { path = "devenv-run-tests" } +tasks = { path = "tasks" } +xtask = { path = "xtask" } + ansiterm = "0.12.2" +blake3 = "1.5.4" clap = { version = "4.5.1", features = ["derive", "cargo"] } cli-table = "0.4.7" dotlock = "0.5.0" fs2 = "0.4.3" +futures = "0.3.30" hex = "0.4.3" include_dir = "0.7.3" indoc = "2.0.4" +lazy_static = "1.5.0" miette = { version = "7.1.0", features = ["fancy"] } nix = { version = "0.28.0", features = ["signal"] } regex = "1.10.3" @@ -29,9 +38,12 @@ schematic = { version = "0.14.3", features = [ ] } serde = "1.0.197" serde_json = "1.0.114" +serde_repr = "0.1.19" serde_yaml = "0.9.32" sha2 = "0.10.8" +sqlx = { version = "0.8.2", features = ["time", "sqlite", "runtime-tokio"] } tempdir = "0.3.7" +thiserror = "1.0.63" tracing = "0.1.40" which = "6.0.0" whoami = "1.5.1" @@ -43,3 +55,11 @@ tokio = { version = "1.39.3", features = [ "rt-multi-thread", ] } schemars = "0.8.16" + +# Always build optimized sqlx-macro to speed up query checks +[profile.dev.package.sqlx-macros] +opt-level = 3 + +[profile.release] +strip = true +lto = "fat" diff --git a/devenv-eval-cache/Cargo.toml b/devenv-eval-cache/Cargo.toml new file mode 100644 index 000000000..373c7b6ed --- /dev/null +++ b/devenv-eval-cache/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "devenv-eval-cache" +# TODO: switch to workspace version +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[dependencies] +blake3.workspace = true +futures.workspace = true +lazy_static.workspace = true +miette.workspace = true +regex.workspace = true +serde.workspace = true +serde_json.workspace = true +serde_repr.workspace = true +sqlx.workspace = true +thiserror.workspace = true +tokio.workspace = true + +[dev-dependencies] +tempdir.workspace = true diff --git a/devenv-eval-cache/migrations/20240906130404_init.sql b/devenv-eval-cache/migrations/20240906130404_init.sql new file mode 100644 index 000000000..334c7989e --- /dev/null +++ b/devenv-eval-cache/migrations/20240906130404_init.sql @@ -0,0 +1,43 @@ +CREATE TABLE IF NOT EXISTS cached_cmd +( + id INTEGER NOT NULL PRIMARY KEY, + raw TEXT NOT NULL, + cmd_hash CHAR(64) NOT NULL UNIQUE, + input_hash CHAR(64) NOT NULL, + output TEXT NOT NULL, + updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')) +); + +CREATE INDEX IF NOT EXISTS idx_cached_cmd_hash ON cached_cmd(cmd_hash); + +CREATE TABLE IF NOT EXISTS file_path +( + id INTEGER NOT NULL PRIMARY KEY, + path BLOB NOT NULL UNIQUE, + is_directory BOOLEAN NOT NULL, + content_hash CHAR(64) NOT NULL, + modified_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')) +); + +CREATE INDEX IF NOT EXISTS idx_file_path ON file_path(path); + +CREATE TABLE IF NOT EXISTS cmd_input_path +( + id INTEGER NOT NULL PRIMARY KEY, + cached_cmd_id INTEGER, + file_path_id INTEGER, + UNIQUE(cached_cmd_id, file_path_id), + FOREIGN KEY(cached_cmd_id) + REFERENCES cached_cmd(id) + ON UPDATE CASCADE + ON DELETE CASCADE, + FOREIGN KEY(file_path_id) + REFERENCES file_path(id) + ON UPDATE CASCADE + ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_cmd_input_path_cached_cmd_id ON cmd_input_path(cached_cmd_id); +CREATE INDEX IF NOT EXISTS idx_cmd_input_path_file_path_id ON cmd_input_path(file_path_id); +CREATE INDEX IF NOT EXISTS idx_cmd_input_path_composite ON cmd_input_path(cached_cmd_id, file_path_id); diff --git a/devenv-eval-cache/src/bin/main.rs b/devenv-eval-cache/src/bin/main.rs new file mode 100644 index 000000000..8c4e76706 --- /dev/null +++ b/devenv-eval-cache/src/bin/main.rs @@ -0,0 +1,17 @@ +use std::process::Command; + +use devenv_eval_cache::{command, db}; + +#[tokio::main] +async fn main() -> Result<(), command::CommandError> { + let database_url = "sqlite:nix-eval-cache.db"; + let pool = db::setup_db(database_url).await?; + + let mut cmd = Command::new("nix"); + cmd.args(["eval", ".#devenv.processes"]); + + let output = command::CachedCommand::new(&pool).output(&mut cmd).await?; + println!("{}", String::from_utf8_lossy(&output.stdout)); + + Ok(()) +} diff --git a/devenv-eval-cache/src/command.rs b/devenv-eval-cache/src/command.rs new file mode 100644 index 000000000..155c1c5aa --- /dev/null +++ b/devenv-eval-cache/src/command.rs @@ -0,0 +1,523 @@ +use futures::future::join_all; +use miette::Diagnostic; +use sqlx::SqlitePool; +use std::io::{self, BufRead, BufReader, Read}; +use std::path::{Path, PathBuf}; +use std::process::{self, Command, Stdio}; +use std::time::{SystemTime, UNIX_EPOCH}; +use thiserror::Error; + +use crate::{ + db, hash, + internal_log::{InternalLog, Verbosity}, + op::Op, +}; + +#[derive(Error, Diagnostic, Debug)] +pub enum CommandError { + #[error(transparent)] + Io(#[from] io::Error), + #[error(transparent)] + Sqlx(#[from] sqlx::Error), + #[error("Nix command failed: {0}")] + NonZeroExitStatus(process::ExitStatus), +} + +pub struct CachedCommand<'a> { + pool: &'a sqlx::SqlitePool, + force_refresh: bool, + extra_paths: Vec, + excluded_paths: Vec, + on_stderr: Option>, +} + +impl<'a> CachedCommand<'a> { + pub fn new(pool: &'a SqlitePool) -> Self { + Self { + pool, + force_refresh: false, + extra_paths: Vec::new(), + excluded_paths: Vec::new(), + on_stderr: None, + } + } + + /// Watch additional paths for changes. + pub fn watch_path>(&mut self, path: P) -> &mut Self { + self.extra_paths.push(path.as_ref().to_path_buf()); + self + } + + /// Remove a path from being watched for changes. + pub fn unwatch_path>(&mut self, path: P) -> &mut Self { + self.excluded_paths.push(path.as_ref().to_path_buf()); + self + } + + /// Force re-evaluation of the command. + pub fn force_refresh(&mut self) -> &mut Self { + self.force_refresh = true; + self + } + + pub fn on_stderr(&mut self, f: F) -> &mut Self + where + F: Fn(&InternalLog) + Send + 'static, + { + self.on_stderr = Some(Box::new(f)); + self + } + + /// Run a (Nix) command with caching enabled. + /// + /// If the command has been run before and the files it depends on have not been modified, + /// the cached output will be returned. + pub async fn output(mut self, cmd: &'a mut Command) -> Result { + let raw_cmd = format!("{:?}", cmd); + let cmd_hash = hash::digest(&raw_cmd); + + // Check whether the command has been previously run and the files it depends on have not been changed. + if !self.force_refresh { + if let Ok(Some(output)) = query_cached_output(self.pool, &cmd_hash).await { + return Ok(output); + } + } + + cmd.arg("-vv") + .arg("--log-format") + .arg("internal-json") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let mut child = cmd.spawn().map_err(CommandError::Io)?; + + let mut stdout = child.stdout.take().unwrap(); + let stderr = child.stderr.take().unwrap(); + + let on_stderr = self.on_stderr.take(); + + let stderr_thread = tokio::spawn(async move { + let reader = BufReader::new(stderr); + let mut raw_lines: Vec = Vec::new(); + let mut ops = Vec::new(); + + let mut lines = reader.lines(); + while let Some(Ok(line)) = lines.next() { + if let Some(log) = InternalLog::parse(&line).and_then(Result::ok) { + if let Some(ref f) = &on_stderr { + f(&log); + } + + // FIX: verbosity + if let Some(msg) = log.get_log_msg_by_level(Verbosity::Info) { + raw_lines.extend_from_slice(msg.as_bytes()); + } + + if let Some(op) = extract_op_from_log_line(log) { + ops.push(op); + } + } + } + + (ops, raw_lines) + }); + + let stdout_thread = tokio::spawn(async move { + let mut output = Vec::new(); + stdout.read_to_end(&mut output).map(|_| output) + }); + + let status = child.wait().map_err(CommandError::Io)?; + + if !status.success() { + return Err(CommandError::NonZeroExitStatus(status)); + } + + let stdout = stdout_thread.await.unwrap().map_err(CommandError::Io)?; + let (mut ops, stderr) = stderr_thread.await.unwrap(); + + // Remove excluded paths if any are a parent directory + ops.retain_mut(|op| { + !self + .excluded_paths + .iter() + .any(|path| op.source().starts_with(path)) + }); + + // Convert Ops to FilePaths + let mut file_path_futures = ops + .into_iter() + .map(|op| { + tokio::task::spawn_blocking(move || { + FilePath::new(op.source().to_path_buf()).map_err(CommandError::Io) + }) + }) + .collect::>(); + + // Watch additional paths + file_path_futures.extend(self.extra_paths.into_iter().map(|path| { + tokio::task::spawn_blocking(move || FilePath::new(path).map_err(CommandError::Io)) + })); + + let mut file_paths = join_all(file_path_futures) + .await + .into_iter() + .flatten() + // TODO: add tracing here + .filter_map(Result::ok) + .collect::>(); + + file_paths.sort_by(|a, b| a.path.cmp(&b.path)); + file_paths.dedup(); + + let input_hash = hash::digest( + &file_paths + .iter() + .map(|p| p.content_hash.clone()) + .collect::(), + ); + + let _ = db::insert_command_with_files( + self.pool, + &raw_cmd, + &cmd_hash, + &input_hash, + &stdout, + &file_paths, + ) + .await + .map_err(CommandError::Sqlx)?; + + Ok(Output { + status, + stdout, + stderr, + paths: file_paths, + }) + } +} + +/// Check whether the command supports the flags required for caching. +pub fn supports_eval_caching(cmd: &Command) -> bool { + cmd.get_program().to_string_lossy().ends_with("nix") +} + +pub struct Output { + /// The status code of the command. + pub status: process::ExitStatus, + /// The data that the process wrote to stdout. + pub stdout: Vec, + /// The data that the process wrote to stderr. + pub stderr: Vec, + /// A list of paths that the command depends on and their hashes. + pub paths: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct FilePath { + pub path: PathBuf, + pub is_directory: bool, + pub content_hash: String, + pub modified_at: SystemTime, +} + +impl FilePath { + pub fn new(path: PathBuf) -> Result { + let is_directory = path.is_dir(); + let content_hash = if is_directory { + let paths = std::fs::read_dir(&path)? + .filter_map(Result::ok) + .map(|entry| entry.path().to_string_lossy().to_string()) + .collect::(); + hash::digest(&paths) + } else { + hash::compute_file_hash(&path)? + }; + let modified_at = path.metadata()?.modified()?; + Ok(Self { + path, + is_directory, + content_hash, + modified_at, + }) + } +} + +impl From for FilePath { + fn from(row: db::FilePathRow) -> Self { + Self { + path: row.path, + is_directory: row.is_directory, + content_hash: row.content_hash, + modified_at: row.modified_at, + } + } +} + +/// Try to fetch the cached output for a hashed command. +/// +/// Returns the cached output if the command has been cached and none of the file dependencies have +/// been updated. +async fn query_cached_output( + pool: &SqlitePool, + cmd_hash: &str, +) -> Result, CommandError> { + let cached_cmd = db::get_command_by_hash(pool, cmd_hash) + .await + .map_err(CommandError::Sqlx)?; + + if let Some(cmd) = cached_cmd { + let mut files = db::get_files_by_command_id(pool, cmd.id) + .await + .map_err(CommandError::Sqlx)?; + + files.sort_by(|a, b| a.path.cmp(&b.path)); + files.dedup(); + + let mut should_refresh = false; + + let file_input_hash = hash::digest( + &files + .iter() + .map(|f| f.content_hash.clone()) + .collect::(), + ); + + // Hash of input hashes do not match + if cmd.input_hash != file_input_hash { + should_refresh = true; + } + + if !should_refresh { + let mut set = tokio::task::JoinSet::new(); + + for file in &files { + let file = file.clone(); + set.spawn_blocking(|| check_file_state(file)); + } + + while let Some(res) = set.join_next().await { + if let Ok(Ok(file_state)) = res { + match file_state { + FileState::MetadataModified { + modified_at, path, .. + } => { + // TODO: batch with query builder? + db::update_file_modified_at(pool, path, modified_at) + .await + .map_err(CommandError::Sqlx)?; + } + FileState::Modified { .. } => { + should_refresh = true; + } + FileState::Removed { .. } => { + should_refresh = true; + } + _ => (), + } + } + } + }; + + if should_refresh { + Ok(None) + } else { + db::update_command_updated_at(pool, cmd.id) + .await + .map_err(CommandError::Sqlx)?; + + // No files have been modified, returning cached output + Ok(Some(Output { + status: process::ExitStatus::default(), + stdout: cmd.output, + stderr: Vec::new(), + paths: files.into_iter().map(FilePath::from).collect(), + })) + } + } else { + Ok(None) + } +} + +/// Convert a parse log line into into an `Op`. +/// Filters out paths that don't impact caching. +fn extract_op_from_log_line(log: InternalLog) -> Option { + match log { + InternalLog::Msg { .. } => Op::from_internal_log(&log).and_then(|op| match op { + Op::EvaluatedFile { ref source } + | Op::ReadFile { ref source } + | Op::ReadDir { ref source } + | Op::CopiedSource { ref source, .. } + | Op::TrackedPath { ref source } + if source.starts_with("/") && !source.starts_with("/nix/store") => + { + Some(op) + } + _ => None, + }), + _ => None, + } +} + +/// Represents the various states of "modified" that we care about. +#[derive(Debug)] +#[allow(dead_code)] +enum FileState { + /// The file has not been modified since it was last cached. + Unchanged { path: PathBuf }, + /// The file's metadata, i.e. timestamp, has changed, but its content remains the same. + MetadataModified { + path: PathBuf, + modified_at: SystemTime, + }, + /// The file's contents have been modified. + Modified { + path: PathBuf, + new_hash: String, + modified_at: SystemTime, + }, + /// The file no longer exists in the file system. + Removed { path: PathBuf }, +} + +fn check_file_state(file: db::FilePathRow) -> io::Result { + let metadata = match std::fs::metadata(&file.path) { + Ok(metadata) => metadata, + // Fix + Err(_) => return Ok(FileState::Removed { path: file.path }), + }; + + let modified_at = metadata.modified().and_then(truncate_to_seconds)?; + if modified_at == file.modified_at { + // File has not been modified + return Ok(FileState::Unchanged { path: file.path }); + } + + // mtime has changed, check if content has changed + let new_hash = if file.is_directory { + if !metadata.is_dir() { + return Ok(FileState::Removed { path: file.path }); + } + + let paths = std::fs::read_dir(&file.path)? + .filter_map(Result::ok) + .map(|entry| entry.path().to_string_lossy().to_string()) + .collect::(); + hash::digest(&paths) + } else { + hash::compute_file_hash(&file.path)? + }; + + if new_hash == file.content_hash { + // File touched but hash unchanged + Ok(FileState::MetadataModified { + path: file.path, + modified_at, + }) + } else { + // Hash has changed, return new hash + Ok(FileState::Modified { + path: file.path, + new_hash, + modified_at, + }) + } +} + +fn truncate_to_seconds(time: SystemTime) -> io::Result { + let duration_since_epoch = time + .duration_since(UNIX_EPOCH) + .map_err(|_| io::Error::new(io::ErrorKind::Other, "SystemTime before UNIX EPOCH"))?; + + let seconds = duration_since_epoch.as_secs(); + Ok(UNIX_EPOCH + std::time::Duration::from_secs(seconds)) +} + +#[cfg(test)] +mod test { + use super::*; + use std::fs::File; + use std::io::Write; + use tempdir::TempDir; + + fn create_file_row(dir: &TempDir, content: &[u8]) -> db::FilePathRow { + let file_path = dir.path().join("test_file.txt"); + let mut file = File::create(&file_path).unwrap(); + file.write_all(content).unwrap(); + + let metadata = file_path.metadata().unwrap(); + let modified_at = metadata.modified().unwrap(); + let truncated_modified_at = truncate_to_seconds(modified_at).unwrap(); + let content_hash = hash::compute_file_hash(&file_path).unwrap(); + + db::FilePathRow { + path: file_path, + is_directory: false, + content_hash, + modified_at: truncated_modified_at, + updated_at: truncated_modified_at, + } + } + + #[test] + fn test_unchanged_file() { + let temp_dir = TempDir::new("test_unchanged_file").unwrap(); + let file_row = create_file_row(&temp_dir, b"Hello, World!"); + + assert!(matches!( + check_file_state(file_row), + Ok(FileState::Unchanged { .. }) + )); + } + + #[test] + fn test_metadata_modified_file() { + let temp_dir = TempDir::new("test_metadata_modified_file").unwrap(); + let file_row = create_file_row(&temp_dir, b"Hello, World!"); + + // Sleep to ensure the new modification time is different + std::thread::sleep(std::time::Duration::from_secs(1)); + + // Update the file's timestamp + let new_time = SystemTime::now(); + let file = File::open(&file_row.path).unwrap(); + file.set_modified(new_time).unwrap(); + drop(file); + + assert!(matches!( + check_file_state(file_row), + Ok(FileState::MetadataModified { .. }) + )); + } + + #[test] + fn test_content_modified_file() { + let temp_dir = TempDir::new("test_content_modified_file").unwrap(); + let file_row = create_file_row(&temp_dir, b"Hello, World!"); + + std::thread::sleep(std::time::Duration::from_secs(1)); + + // Modify the file contents + let mut file = File::create(&file_row.path).unwrap(); + file.write_all(b"Modified content").unwrap(); + + assert!(matches!( + check_file_state(file_row), + Ok(FileState::Modified { .. }) + )); + } + + #[test] + fn test_removed_file() { + let temp_dir = TempDir::new("test_removed_file").unwrap(); + let file_row = create_file_row(&temp_dir, b"Hello, World!"); + + // Remove the file + std::fs::remove_file(&file_row.path).unwrap(); + + assert!(matches!( + check_file_state(file_row), + Ok(FileState::Removed { .. }) + )); + } +} diff --git a/devenv-eval-cache/src/db.rs b/devenv-eval-cache/src/db.rs new file mode 100644 index 000000000..0432f134a --- /dev/null +++ b/devenv-eval-cache/src/db.rs @@ -0,0 +1,607 @@ +use super::command::FilePath; +use sqlx::sqlite::{Sqlite, SqliteConnectOptions, SqliteJournalMode, SqliteRow, SqliteSynchronous}; +use sqlx::{Acquire, Row, SqlitePool}; +use std::ffi::OsStr; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub async fn setup_db>(database_url: P) -> Result { + let conn_options = SqliteConnectOptions::from_str(database_url.as_ref())? + .foreign_keys(true) + .journal_mode(SqliteJournalMode::Wal) + .synchronous(SqliteSynchronous::Normal) + .pragma("mmap_size", "134217728") + .pragma("journal_size_limit", "27103364") + .pragma("cache_size", "2000") + .create_if_missing(true); + + let pool = SqlitePool::connect_with(conn_options).await?; + + sqlx::migrate!("./migrations").run(&pool).await?; + + Ok(pool) +} + +/// The row type for the `cached_cmd` table. +#[derive(Clone, Debug)] +pub struct CommandRow { + /// The primary key + pub id: i64, + /// The raw command string (for debugging) + pub raw: String, + /// A hash of the command string + pub cmd_hash: String, + /// A hash of the content hashes of the input files + pub input_hash: String, + /// The raw output of the command + pub output: Vec, + /// The time the cached command was checked or created + pub updated_at: SystemTime, +} + +impl sqlx::FromRow<'_, SqliteRow> for CommandRow { + fn from_row(row: &SqliteRow) -> Result { + let id: i64 = row.get("id"); + let raw: String = row.get("raw"); + let cmd_hash: String = row.get("cmd_hash"); + let input_hash: String = row.get("input_hash"); + let output: Vec = row.get("output"); + let updated_at: u64 = row.get("updated_at"); + Ok(Self { + id, + raw, + cmd_hash, + input_hash, + output, + updated_at: UNIX_EPOCH + std::time::Duration::from_secs(updated_at), + }) + } +} + +pub async fn get_command_by_hash<'a, A>( + conn: A, + cmd_hash: &str, +) -> Result, sqlx::Error> +where + A: Acquire<'a, Database = Sqlite>, +{ + let mut conn = conn.acquire().await?; + + let record = sqlx::query_as( + r#" + SELECT * + FROM cached_cmd + WHERE cmd_hash = ? + "#, + ) + .bind(cmd_hash) + .fetch_optional(&mut *conn) + .await?; + + Ok(record) +} + +pub async fn insert_command_with_files<'a, A>( + conn: A, + raw_cmd: &str, + cmd_hash: &str, + input_hash: &str, + output: &[u8], + paths: &[FilePath], +) -> Result<(i64, Vec), sqlx::Error> +where + A: Acquire<'a, Database = Sqlite>, +{ + let mut conn = conn.acquire().await?; + let mut tx = conn.begin().await?; + + delete_command(&mut tx, cmd_hash).await?; + let command_id = insert_command(&mut tx, raw_cmd, cmd_hash, input_hash, output).await?; + let file_ids = insert_files(&mut tx, paths, command_id).await?; + + tx.commit().await?; + + Ok((command_id, file_ids)) +} + +async fn insert_command<'a, A>( + conn: A, + raw_cmd: &str, + cmd_hash: &str, + input_hash: &str, + output: &[u8], +) -> Result +where + A: Acquire<'a, Database = Sqlite>, +{ + let mut conn = conn.acquire().await?; + + let record = sqlx::query!( + r#" + INSERT INTO cached_cmd (raw, cmd_hash, input_hash, output) + VALUES (?, ?, ?, ?) + RETURNING id + "#, + raw_cmd, + cmd_hash, + input_hash, + output + ) + .fetch_one(&mut *conn) + .await?; + + Ok(record.id) +} + +async fn delete_command<'a, A>(conn: A, cmd_hash: &str) -> Result<(), sqlx::Error> +where + A: Acquire<'a, Database = Sqlite>, +{ + let mut conn = conn.acquire().await?; + + sqlx::query!( + r#" + DELETE FROM cached_cmd + WHERE cmd_hash = ? + "#, + cmd_hash + ) + .execute(&mut *conn) + .await?; + + Ok(()) +} + +pub async fn update_command_updated_at<'a, A>(conn: A, id: i64) -> Result<(), sqlx::Error> +where + A: Acquire<'a, Database = Sqlite>, +{ + let mut conn = conn.acquire().await?; + + sqlx::query!( + r#" + UPDATE cached_cmd + SET updated_at = strftime('%s', 'now') + WHERE id = ? + "#, + id + ) + .execute(&mut *conn) + .await?; + + Ok(()) +} + +async fn insert_files<'a, A>( + conn: A, + paths: &[FilePath], + command_id: i64, +) -> Result, sqlx::Error> +where + A: Acquire<'a, Database = Sqlite>, +{ + let mut conn = conn.acquire().await?; + + let file_path_query = r#" + INSERT INTO file_path (path, is_directory, content_hash, modified_at) + VALUES (?, ?, ?, ?) + ON CONFLICT (path) DO UPDATE + SET content_hash = excluded.content_hash, + is_directory = excluded.is_directory, + modified_at = excluded.modified_at, + updated_at = strftime('%s', 'now') + RETURNING id + "#; + + let mut file_ids = Vec::with_capacity(paths.len()); + for FilePath { + path, + is_directory, + content_hash, + modified_at, + } in paths + { + let modified_at = modified_at.duration_since(UNIX_EPOCH).unwrap().as_secs() as i64; + let id: i64 = sqlx::query(file_path_query) + .bind(path.to_path_buf().into_os_string().as_bytes()) + .bind(is_directory) + .bind(content_hash) + .bind(modified_at) + .fetch_one(&mut *conn) + .await? + .get(0); + file_ids.push(id); + } + + let cmd_input_path_query = r#" + INSERT INTO cmd_input_path (cached_cmd_id, file_path_id) + VALUES (?, ?) + ON CONFLICT (cached_cmd_id, file_path_id) DO NOTHING + "#; + + for &file_id in &file_ids { + sqlx::query(cmd_input_path_query) + .bind(command_id) + .bind(file_id) + .execute(&mut *conn) + .await?; + } + Ok(file_ids) +} + +/// The row type for the `file_path` table. +#[derive(Clone, Debug, PartialEq)] +pub struct FilePathRow { + /// A path + pub path: PathBuf, + pub is_directory: bool, + /// The hash of the file's content + pub content_hash: String, + /// The last modified time of the file + pub modified_at: SystemTime, + /// The last time the row was updated + pub updated_at: SystemTime, +} + +impl sqlx::FromRow<'_, SqliteRow> for FilePathRow { + fn from_row(row: &SqliteRow) -> Result { + let path: &[u8] = row.get("path"); + let is_directory: bool = row.get("is_directory"); + let content_hash: String = row.get("content_hash"); + let modified_at: u64 = row.get("modified_at"); + let updated_at: u64 = row.get("updated_at"); + Ok(Self { + path: PathBuf::from(OsStr::from_bytes(path)), + is_directory, + content_hash, + modified_at: UNIX_EPOCH + std::time::Duration::from_secs(modified_at), + updated_at: UNIX_EPOCH + std::time::Duration::from_secs(updated_at), + }) + } +} + +pub async fn get_files_by_command_id( + pool: &SqlitePool, + command_id: i64, +) -> Result, sqlx::Error> { + let files = sqlx::query_as( + r#" + SELECT fp.path, fp.is_directory, fp.content_hash, fp.modified_at, fp.updated_at + FROM file_path fp + JOIN cmd_input_path cip ON fp.id = cip.file_path_id + WHERE cip.cached_cmd_id = ? + "#, + ) + .bind(command_id) + .fetch_all(pool) + .await?; + + Ok(files) +} + +pub async fn get_files_by_command_hash( + pool: &SqlitePool, + command_hash: &str, +) -> Result, sqlx::Error> { + let files = sqlx::query_as( + r#" + SELECT fp.path, fp.is_directory, fp.content_hash, fp.modified_at, fp.updated_at + FROM file_path fp + JOIN cmd_input_path cip ON fp.id = cip.file_path_id + JOIN cached_cmd cc ON cip.cached_cmd_id = cc.id + WHERE cc.cmd_hash = ? + "#, + ) + .bind(command_hash) + .fetch_all(pool) + .await?; + + Ok(files) +} + +pub async fn update_file_modified_at>( + pool: &SqlitePool, + path: P, + modified_at: SystemTime, +) -> Result<(), sqlx::Error> { + let modified_at = modified_at.duration_since(UNIX_EPOCH).unwrap().as_secs() as i64; + + sqlx::query( + r#" + UPDATE file_path + SET modified_at = ?, updated_at = strftime('%s', 'now') + WHERE path = ? + "#, + ) + .bind(modified_at) + .bind(path.as_ref().to_path_buf().into_os_string().as_bytes()) + .execute(pool) + .await?; + + Ok(()) +} + +pub async fn delete_unreferenced_files(pool: &SqlitePool) -> Result { + let result = sqlx::query( + r#" + DELETE FROM file_path + WHERE NOT EXISTS ( + SELECT 1 + FROM cmd_input_path + WHERE cmd_input_path.file_path_id = file_path.id + ) + "#, + ) + .execute(pool) + .await?; + + Ok(result.rows_affected()) +} + +#[cfg(test)] +mod tests { + use crate::hash; + + use super::*; + use sqlx::SqlitePool; + + #[sqlx::test] + async fn test_insert_and_retrieve_command(pool: SqlitePool) { + let raw_cmd = "nix-build -A hello"; + let cmd_hash = hash::digest(raw_cmd); + let output = b"Hello, world!"; + let modified_at = SystemTime::now(); + let paths = vec![ + FilePath { + path: "/path/to/file1".into(), + is_directory: false, + content_hash: "hash1".to_string(), + modified_at, + }, + FilePath { + path: "/path/to/file2".into(), + is_directory: false, + content_hash: "hash2".to_string(), + modified_at, + }, + ]; + let input_hash = hash::digest( + &paths + .iter() + .map(|fp| fp.content_hash.clone()) + .collect::(), + ); + + let (command_id, file_ids) = + insert_command_with_files(&pool, raw_cmd, &cmd_hash, &input_hash, output, &paths) + .await + .unwrap(); + + assert_eq!(file_ids.len(), 2); + + let retrieved_command = get_command_by_hash(&pool, &cmd_hash) + .await + .unwrap() + .unwrap(); + assert_eq!(retrieved_command.raw, raw_cmd); + assert_eq!(retrieved_command.cmd_hash, cmd_hash); + assert_eq!(retrieved_command.output, output); + + let files = get_files_by_command_id(&pool, command_id).await.unwrap(); + assert_eq!(files.len(), 2); + assert_eq!(files[0].path, PathBuf::from("/path/to/file1")); + assert_eq!(files[0].content_hash, "hash1"); + assert_eq!(files[1].path, PathBuf::from("/path/to/file2")); + assert_eq!(files[1].content_hash, "hash2"); + } + + #[sqlx::test] + async fn test_insert_multiple_commands(pool: SqlitePool) { + // First command + let raw_cmd1 = "nix-build -A hello"; + let cmd_hash1 = hash::digest(raw_cmd1); + let output1 = b"Hello, world!"; + let modified_at = SystemTime::now(); + let paths1 = vec![ + FilePath { + path: "/path/to/file1".into(), + is_directory: false, + content_hash: "hash1".to_string(), + modified_at, + }, + FilePath { + path: "/path/to/file2".into(), + is_directory: false, + content_hash: "hash2".to_string(), + modified_at, + }, + ]; + let input_hash1 = hash::digest( + &paths1 + .iter() + .map(|p| p.content_hash.clone()) + .collect::(), + ); + + let (command_id1, file_ids1) = + insert_command_with_files(&pool, raw_cmd1, &cmd_hash1, &input_hash1, output1, &paths1) + .await + .unwrap(); + + // Second command + let raw_cmd2 = "nix-build -A goodbye"; + let cmd_hash2 = hash::digest(raw_cmd2); + let output2 = b"Goodbye, world!"; + let modified_at = SystemTime::now(); + let paths2 = vec![ + FilePath { + path: "/path/to/file2".into(), + is_directory: false, + content_hash: "hash2".to_string(), + modified_at, + }, + FilePath { + path: "/path/to/file3".into(), + is_directory: false, + content_hash: "hash3".to_string(), + modified_at, + }, + ]; + let input_hash2 = hash::digest( + &paths2 + .iter() + .map(|p| p.content_hash.clone()) + .collect::(), + ); + + let (command_id2, file_ids2) = + insert_command_with_files(&pool, raw_cmd2, &cmd_hash2, &input_hash2, output2, &paths2) + .await + .unwrap(); + + // Verify first command + let retrieved_command1 = get_command_by_hash(&pool, &cmd_hash1) + .await + .unwrap() + .unwrap(); + assert_eq!(retrieved_command1.raw, raw_cmd1); + let files1 = get_files_by_command_id(&pool, command_id1).await.unwrap(); + assert_eq!(files1.len(), 2); + + // Verify second command + let retrieved_command2 = get_command_by_hash(&pool, &cmd_hash2) + .await + .unwrap() + .unwrap(); + assert_eq!(retrieved_command2.raw, raw_cmd2); + let files2 = get_files_by_command_id(&pool, command_id2).await.unwrap(); + assert_eq!(files2.len(), 2); + + // Verify cmd_input_path rows + let all_files = sqlx::query("SELECT * FROM cmd_input_path") + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!(all_files.len(), 4); // 2 files for each command + + // Verify file reuse + assert_eq!(file_ids1.len(), 2); + assert_eq!(file_ids2.len(), 2); + assert!(file_ids1.contains(&file_ids2[0])); // file2 is shared between commands + } + + #[sqlx::test] + async fn test_insert_command_with_modified_files(pool: SqlitePool) { + // First command + let raw_cmd = "nix-build -A hello"; + let cmd_hash = hash::digest(raw_cmd); + let output = b"Hello, world!"; + let modified_at = SystemTime::now(); + let paths1 = vec![ + FilePath { + path: "/path/to/file1".into(), + is_directory: false, + content_hash: "hash1".to_string(), + modified_at, + }, + FilePath { + path: "/path/to/file2".into(), + is_directory: false, + content_hash: "hash2".to_string(), + modified_at, + }, + ]; + let input_hash = hash::digest( + &paths1 + .iter() + .map(|p| p.content_hash.clone()) + .collect::(), + ); + + let (_command_id1, file_ids1) = + insert_command_with_files(&pool, raw_cmd, &cmd_hash, &input_hash, output, &paths1) + .await + .unwrap(); + + // Second command + let paths2 = vec![ + FilePath { + path: "/path/to/file2".into(), + is_directory: false, + content_hash: "hash2".to_string(), + modified_at, + }, + FilePath { + path: "/path/to/file3".into(), + is_directory: false, + content_hash: "hash3".to_string(), + modified_at, + }, + ]; + let input_hash2 = hash::digest( + &paths2 + .iter() + .map(|p| p.content_hash.clone()) + .collect::(), + ); + + let (command_id2, file_ids2) = + insert_command_with_files(&pool, raw_cmd, &cmd_hash, &input_hash2, output, &paths2) + .await + .unwrap(); + + // Investigate the files associated with the new command + let files = get_files_by_command_id(&pool, command_id2).await.unwrap(); + println!( + "Number of files associated with the command: {}", + files.len() + ); + for file in &files { + println!("File path: {:?}, hash: {}", file.path, file.content_hash); + } + + // Check if files are being accumulated instead of replaced + assert_eq!(files.len(), 2, "Expected 2 files, but found {}. Files might be accumulating instead of being replaced.", files.len()); + + // Verify the correct files are associated + let file_paths: Vec<_> = files.iter().map(|f| f.path.to_str().unwrap()).collect(); + assert!( + file_paths.contains(&"/path/to/file2"), + "Expected /path/to/file2 to be present" + ); + assert!( + file_paths.contains(&"/path/to/file3"), + "Expected /path/to/file3 to be present" + ); + assert!( + !file_paths.contains(&"/path/to/file1"), + "Expected /path/to/file1 to be absent" + ); + + // Verify that file2 is reused and file3 is new + assert_eq!(file_ids2.len(), 2, "Expected 2 file IDs"); + assert!( + file_ids1.contains(&file_ids2[0]), + "Expected file2 to be reused" + ); + assert!( + !file_ids1.contains(&file_ids2[1]), + "Expected file3 to be new" + ); + + // Verify that the new command has the correct files + let files = get_files_by_command_id(&pool, command_id2).await.unwrap(); + assert_eq!(files.len(), 2); + assert_eq!(files[0].path, PathBuf::from("/path/to/file2")); + assert_eq!(files[0].content_hash, "hash2"); + assert_eq!(files[1].path, PathBuf::from("/path/to/file3")); + assert_eq!(files[1].content_hash, "hash3"); + + // Verify that file2 is reused and file3 is new + assert_eq!(file_ids2.len(), 2); + assert!(file_ids1.contains(&file_ids2[0])); // file2 is reused + assert!(!file_ids1.contains(&file_ids2[1])); // file3 is new + } +} diff --git a/devenv-eval-cache/src/hash.rs b/devenv-eval-cache/src/hash.rs new file mode 100644 index 000000000..9651c01c4 --- /dev/null +++ b/devenv-eval-cache/src/hash.rs @@ -0,0 +1,14 @@ +use std::path::Path; +use std::{fs, io}; + +pub(crate) fn digest(input: &str) -> String { + let hash = blake3::hash(input.as_bytes()); + hash.to_hex().as_str().to_string() +} + +pub(crate) fn compute_file_hash>(path: P) -> io::Result { + let mut file = fs::File::open(path)?; + let mut hasher = blake3::Hasher::new(); + io::copy(&mut file, &mut hasher)?; + Ok(hasher.finalize().to_hex().as_str().to_string()) +} diff --git a/devenv-eval-cache/src/internal_log.rs b/devenv-eval-cache/src/internal_log.rs new file mode 100644 index 000000000..537dc58b4 --- /dev/null +++ b/devenv-eval-cache/src/internal_log.rs @@ -0,0 +1,273 @@ +use serde::Deserialize; +use serde_repr::Deserialize_repr; +use std::fmt::{self, Display, Formatter}; + +/// Represents Nix's JSON structured log format (--log-format=internal-json). +/// +/// See https://github.com/NixOS/nix/blob/a1cc362d9d249b95e4c9ad403f1e6e26ca302413/src/libutil/logging.cc#L173 +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", tag = "action")] +pub enum InternalLog { + Msg { + level: Verbosity, + msg: String, + // Raw message when logging ErrorInfo + raw_msg: Option, + }, + Start { + id: u64, + level: Verbosity, + #[serde(rename = "type")] + typ: ActivityType, + text: String, + parent: u64, + fields: Vec, + }, + Stop { + id: u64, + }, + Result { + id: u64, + #[serde(rename = "type")] + typ: ResultType, + fields: Vec, + }, + // Possibly deprecated. + SetPhase { + phase: String, + }, +} + +impl InternalLog { + // TODO: assumes UTF-8 encoding + pub fn parse(line: T) -> Option> + where + T: AsRef, + { + line.as_ref() + .strip_prefix("@nix ") + .map(serde_json::from_str) + } + + pub fn get_log_msg_by_level(&self, target_log_level: Verbosity) -> Option { + use std::fmt::Write; + + match self { + // A lot of build messages are tagged as level 0 (Error), making it difficult + // to filter things out. Our hunch is that these messages are coming from the + // nix daemon. + InternalLog::Msg { msg, level, .. } + if *level == Verbosity::Error && self.is_nix_error() => + { + Some(msg.clone()) + } + InternalLog::Msg { msg, level, .. } + if *level > Verbosity::Error && *level <= target_log_level => + { + Some(msg.clone()) + } + + InternalLog::Start { level, text, .. } if *level <= target_log_level => { + Some(text.clone()) + } + InternalLog::Result { + typ: ResultType::BuildLogLine, + fields, + .. + } if target_log_level >= Verbosity::Info => { + let mut msg = String::new(); + for field in fields { + writeln!(msg, "{}", field).ok(); + } + Some(msg.trim_end().to_string()) + } + _ => None, + } + } + + /// Check if the log is an actual error message. + /// + /// In additional to checking the verbosity level of the message, we look for the `error:` prefix in the message. + /// Most messages during the builds (probably from the nix-daemon) are incorrectly logged as errors. + pub fn is_nix_error(&self) -> bool { + if let InternalLog::Msg { + level: Verbosity::Error, + msg, + .. + } = self + { + if msg.starts_with("\u{1b}[31;1merror:") { + return true; + } + } + + false + } +} + +/// See https://github.com/NixOS/nix/blob/322d2c767f2a3f8ef2ac3d1ba46c19caf9a1ffce/src/libutil/error.hh#L33-L42 +#[derive(Copy, Clone, Debug, Default, Deserialize_repr, PartialEq, Eq, PartialOrd, Ord)] +#[repr(u8)] +pub enum Verbosity { + Error = 0, + Warn = 1, + Notice = 2, + #[default] + Info = 3, + Talkative = 4, + Chatty = 5, + Debug = 6, + Vomit = 7, +} + +/// See https://github.com/NixOS/nix/blob/a5959aa12170fc75cafc9e2416fae9aa67f91e6b/src/libutil/logging.hh#L11-L26 +#[derive(Copy, Clone, Debug, Deserialize_repr, PartialEq, Eq, PartialOrd, Ord)] +#[repr(u8)] +pub enum ActivityType { + Unknown = 0, + CopyPath = 100, + FileTransfer = 101, + Realise = 102, + CopyPaths = 103, + Builds = 104, + Build = 105, + OptimiseStore = 106, + VerifyPaths = 107, + Substitute = 108, + QueryPathInfo = 109, + PostBuildHook = 110, + BuildWaiting = 111, + FetchTree = 112, +} + +/// See https://github.com/NixOS/nix/blob/a5959aa12170fc75cafc9e2416fae9aa67f91e6b/src/libutil/logging.hh#L28-L38 +#[derive(Copy, Clone, Debug, Deserialize_repr, PartialEq, Eq, PartialOrd, Ord)] +#[repr(u8)] +pub enum ResultType { + FileLinked = 100, + BuildLogLine = 101, + UntrustedPath = 102, + CorruptedPath = 103, + SetPhase = 104, + Progress = 105, + SetExpected = 106, + PostBuildLogLine = 107, + FetchStatus = 108, +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(untagged)] +pub enum Field { + Int(u64), + String(String), +} + +impl Display for Field { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + Field::Int(i) => write!(f, "{}", i), + Field::String(s) => write!(f, "{}", s), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_parse_log_msg() { + let line = r#"@nix {"action":"msg","level":1,"msg":"hello"}"#; + let log = InternalLog::parse(line).unwrap().unwrap(); + assert_eq!( + log, + InternalLog::Msg { + level: Verbosity::Warn, + msg: "hello".to_string(), + raw_msg: None, + } + ); + } + + #[test] + fn test_parse_log_start() { + let line = r#"@nix {"action":"start","id":1,"level":3,"type":100,"text":"hello","parent":0,"fields":[]}"#; + let log = InternalLog::parse(line).unwrap().unwrap(); + assert_eq!( + log, + InternalLog::Start { + id: 1, + level: Verbosity::Info, + typ: ActivityType::CopyPath, + text: "hello".to_string(), + parent: 0, + fields: vec![], + } + ); + } + + #[test] + fn test_parse_log_stop() { + let line = r#"@nix {"action":"stop","id":1}"#; + let log = InternalLog::parse(line).unwrap().unwrap(); + assert_eq!(log, InternalLog::Stop { id: 1 }); + } + + #[test] + fn test_parse_log_result() { + let line = r#"@nix {"action":"result","id":1,"type":101,"fields":["hello"]}"#; + let log = InternalLog::parse(line).unwrap().unwrap(); + assert_eq!( + log, + InternalLog::Result { + id: 1, + typ: ResultType::BuildLogLine, + fields: vec![Field::String("hello".to_string())], + } + ); + } + + #[test] + fn test_parse_invalid_log() { + let line = r#"@nix {"action":"invalid"}"#; + assert!(InternalLog::parse(line).unwrap().is_err()); + } + + #[test] + fn test_parse_non_nix_log() { + let line = "This is not a Nix log line"; + assert!(InternalLog::parse(line).is_none()); + } + + #[test] + fn test_verbosity_deserialize() { + let json = r#"0"#; + let verbosity: Verbosity = serde_json::from_str(json).unwrap(); + assert_eq!(verbosity, Verbosity::Error); + } + + #[test] + // Ensure that only messages containing the prefix `error:` are detected as error messages. + // See `is_nix_error` for more details. + fn test_is_nix_error() { + let log = InternalLog::Msg { + level: Verbosity::Error, + msg: "\u{1b}[31;1merror:\u{1b}[0m\nsomething went wrong".to_string(), + raw_msg: None, + }; + eprintln!("{:?}", log); + assert!(log.is_nix_error()); + } + + #[test] + // Ensure we don't interpret non-error messages as errors. + // See `is_nix_error` for more details. + fn test_is_nix_error_misleveled_msgs() { + let log = InternalLog::Msg { + level: Verbosity::Error, + msg: "not an error".to_string(), + raw_msg: None, + }; + assert!(!log.is_nix_error()); + } +} diff --git a/devenv-eval-cache/src/lib.rs b/devenv-eval-cache/src/lib.rs new file mode 100644 index 000000000..3d910152d --- /dev/null +++ b/devenv-eval-cache/src/lib.rs @@ -0,0 +1,8 @@ +pub mod command; +pub mod db; +pub(crate) mod hash; +pub mod internal_log; +pub mod op; + +pub use command::{supports_eval_caching, CachedCommand, Output}; +pub use db::setup_db; diff --git a/devenv-eval-cache/src/op.rs b/devenv-eval-cache/src/op.rs new file mode 100644 index 000000000..b7cd69c87 --- /dev/null +++ b/devenv-eval-cache/src/op.rs @@ -0,0 +1,165 @@ +use crate::internal_log::InternalLog; + +use regex::Regex; +use std::path::PathBuf; + +/// A sum-type of filesystem operations that we can extract from the Nix logs. +#[derive(Clone, Debug, PartialEq)] +pub enum Op { + /// Copied a file to the Nix store. + CopiedSource { source: PathBuf, target: PathBuf }, + /// Evaluated a Nix file. + EvaluatedFile { source: PathBuf }, + /// Read a file's contents with `builtins.readFile`. + ReadFile { source: PathBuf }, + /// List a directory's contents with `builtins.readDir`. + ReadDir { source: PathBuf }, + /// Used a tracked devenv string path. + TrackedPath { source: PathBuf }, +} + +impl Op { + /// Extract an `Op` from a `InternalLog`. + pub fn from_internal_log(log: &InternalLog) -> Option { + lazy_static::lazy_static! { + static ref EVALUATED_FILE: Regex = + Regex::new("^evaluating file '(?P.*)'$").expect("invalid regex"); + static ref COPIED_SOURCE: Regex = + Regex::new("^copied source '(?P.*)' -> '(?P.*)'$").expect("invalid regex"); + static ref READ_FILE: Regex = + Regex::new("^devenv readFile: '(?P.*)'$").expect("invalid regex"); + static ref READ_DIR: Regex = + Regex::new("^devenv readDir: '(?P.*)'$").expect("invalid regex"); + static ref TRACKED_PATH: Regex = + Regex::new("^trace: devenv path: '(?P.*)'$").expect("invalid regex"); + } + + match log { + InternalLog::Msg { msg, .. } => { + if let Some(matches) = COPIED_SOURCE.captures(msg) { + let source = PathBuf::from(&matches["source"]); + let target = PathBuf::from(&matches["target"]); + Some(Op::CopiedSource { source, target }) + } else if let Some(matches) = EVALUATED_FILE.captures(msg) { + let mut source = PathBuf::from(&matches["source"]); + // If the evaluated file is a directory, we assume that the file is `default.nix`. + if source.is_dir() { + source.push("default.nix"); + } + Some(Op::EvaluatedFile { source }) + } else if let Some(matches) = READ_FILE.captures(msg) { + let source = PathBuf::from(&matches["source"]); + Some(Op::ReadFile { source }) + } else if let Some(matches) = READ_DIR.captures(msg) { + let source = PathBuf::from(&matches["source"]); + Some(Op::ReadDir { source }) + } else if let Some(matches) = TRACKED_PATH.captures(msg) { + let source = PathBuf::from(&matches["source"]); + Some(Op::TrackedPath { source }) + } else { + None + } + } + _ => None, + } + } + + pub fn source(&self) -> &PathBuf { + match self { + Op::CopiedSource { source, .. } => source, + Op::EvaluatedFile { source } => source, + Op::ReadFile { source } => source, + Op::ReadDir { source } => source, + Op::TrackedPath { source } => source, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::internal_log::Verbosity; + + fn create_log(msg: &str) -> InternalLog { + InternalLog::Msg { + msg: msg.to_string(), + raw_msg: None, + level: Verbosity::Warn, + } + } + + #[test] + fn test_copied_source() { + let log = create_log("copied source '/path/to/source' -> '/path/to/target'"); + let op = Op::from_internal_log(&log); + assert_eq!( + op, + Some(Op::CopiedSource { + source: PathBuf::from("/path/to/source"), + target: PathBuf::from("/path/to/target"), + }) + ); + } + + #[test] + fn test_evaluated_file() { + let log = create_log("evaluating file '/path/to/file'"); + let op = Op::from_internal_log(&log); + assert_eq!( + op, + Some(Op::EvaluatedFile { + source: PathBuf::from("/path/to/file"), + }) + ); + } + + #[test] + fn test_read_file() { + let log = create_log("devenv readFile: '/path/to/file'"); + let op = Op::from_internal_log(&log); + assert_eq!( + op, + Some(Op::ReadFile { + source: PathBuf::from("/path/to/file"), + }) + ); + } + + #[test] + fn test_read_dir() { + let log = create_log("devenv readDir: '/path/to/dir'"); + let op = Op::from_internal_log(&log); + assert_eq!( + op, + Some(Op::ReadDir { + source: PathBuf::from("/path/to/dir"), + }) + ); + } + + #[test] + fn test_tracked_path() { + let log = create_log("trace: devenv path: '/path/to/file'"); + let op = Op::from_internal_log(&log); + assert_eq!( + op, + Some(Op::TrackedPath { + source: PathBuf::from("/path/to/file"), + }) + ); + } + + #[test] + fn test_unmatched_log() { + let log = create_log("some unrelated message"); + let op = Op::from_internal_log(&log); + assert_eq!(op, None); + } + + #[test] + fn test_non_msg_log() { + let log = InternalLog::Stop { id: 1 }; + let op = Op::from_internal_log(&log); + assert_eq!(op, None); + } +} diff --git a/devenv-run-tests/Cargo.toml b/devenv-run-tests/Cargo.toml index b13c40344..14e68207b 100644 --- a/devenv-run-tests/Cargo.toml +++ b/devenv-run-tests/Cargo.toml @@ -6,7 +6,6 @@ license.workspace = true [dependencies] clap.workspace = true +devenv.workspace = true tempdir.workspace = true - -devenv= { path = "../devenv" } tokio = "1.39.3" diff --git a/devenv-run-tests/src/main.rs b/devenv-run-tests/src/main.rs index 2e81ba677..64f4e2396 100644 --- a/devenv-run-tests/src/main.rs +++ b/devenv-run-tests/src/main.rs @@ -95,8 +95,7 @@ async fn run_tests_in_directory( devenv_dotfile: Some(tmpdir.path().to_path_buf()), ..Default::default() }; - let mut devenv = Devenv::new(options); - devenv.create_directories()?; + let mut devenv = Devenv::new(options).await; // A script to patch files in the working directory before the shell. let patch_script = ".patch.sh"; diff --git a/devenv.nix b/devenv.nix index 31e49014b..46c73e004 100644 --- a/devenv.nix +++ b/devenv.nix @@ -4,6 +4,7 @@ env.BROWSERSLIST_IGNORE_OLD_DATA = "1"; env.RUST_LOG = "devenv=debug"; env.RUST_LOG_SPAN_EVENTS = "full"; + env.DATABASE_URL = "sqlite:.devenv/nix-eval-cache.db"; packages = [ pkgs.cairo @@ -13,6 +14,7 @@ pkgs.tesh pkgs.watchexec pkgs.openssl + pkgs.sqlx-cli ] ++ lib.optionals pkgs.stdenv.isDarwin (with pkgs.darwin.apple_sdk; [ frameworks.SystemConfiguration ]); diff --git a/devenv/Cargo.toml b/devenv/Cargo.toml index 4b0731da6..044eb6801 100644 --- a/devenv/Cargo.toml +++ b/devenv/Cargo.toml @@ -20,8 +20,8 @@ include_dir.workspace = true indoc.workspace = true miette.workspace = true nix.workspace = true +devenv-eval-cache.workspace = true petgraph = "0.6.5" -pretty_assertions = { version = "1.4.0", features = ["unstable"] } regex.workspace = true reqwest.workspace = true schemars.workspace = true @@ -30,12 +30,16 @@ serde.workspace = true serde_json.workspace = true serde_yaml.workspace = true sha2.workspace = true +sqlx.workspace = true tempdir.workspace = true tempfile = "3.12.0" -test-log = { version = "0.2.16", features = ["trace"] } thiserror = "1.0.63" tokio.workspace = true tracing.workspace = true which.workspace = true whoami.workspace = true xdg.workspace = true + +[dev-dependencies] +pretty_assertions = { version = "1.4.0", features = ["unstable"] } +test-log = { version = "0.2.16", features = ["trace"] } diff --git a/devenv/src/cli.rs b/devenv/src/cli.rs index a6d1ef503..7b57fa3c6 100644 --- a/devenv/src/cli.rs +++ b/devenv/src/cli.rs @@ -10,15 +10,24 @@ use std::path::PathBuf; dont_delimit_trailing_values = true, about = format!("https://devenv.sh {}: Fast, Declarative, Reproducible, and Composable Developer Environments", crate_version!()) )] -pub(crate) struct Cli { +pub struct Cli { #[command(subcommand)] - pub(crate) command: Commands, + pub command: Commands, #[command(flatten)] - pub(crate) global_options: GlobalOptions, + pub global_options: GlobalOptions, } -#[derive(Parser, Clone)] +impl Cli { + /// Parse the CLI arguments with clap and resolve any conflicting options. + pub fn parse_and_resolve_options() -> Self { + let mut cli = Self::parse(); + cli.global_options.resolve_overrides(); + cli + } +} + +#[derive(Clone, Debug, Parser)] pub struct GlobalOptions { #[arg(short, long, global = true, help = "Enable debug log level.")] pub verbose: bool, @@ -40,7 +49,7 @@ pub struct GlobalOptions { #[arg( short = 'u', long, - help = "Maximum number CPU cores being used by a single build..", + help = "Maximum number CPU cores being used by a single build.", default_value = "2" )] pub cores: u8, @@ -56,9 +65,25 @@ pub struct GlobalOptions { )] pub impure: bool, - #[arg(long, global = true, help = "Use flake cache for evaluation results.")] + #[arg(long, global = true, help = "Cache the results of Nix evaluation.")] + #[arg( + long_help = "Cache the results of Nix evaluation. Use --no-eval-cache to disable caching." + )] + #[arg(default_value_t = true, overrides_with = "no_eval_cache")] pub eval_cache: bool, + /// Disable the evaluation cache. Sets `eval_cache` to false. + #[arg(long, global = true, hide = true)] + #[arg(overrides_with = "eval_cache")] + no_eval_cache: bool, + + #[arg( + long, + global = true, + help = "Force a refresh of the Nix evaluation cache." + )] + pub refresh_eval_cache: bool, + #[arg( long, global = true, @@ -110,7 +135,9 @@ impl Default for GlobalOptions { cores: 2, system: default_system(), impure: false, - eval_cache: false, + eval_cache: true, + no_eval_cache: false, + refresh_eval_cache: false, offline: false, clean: None, nix_debugger: false, @@ -120,8 +147,18 @@ impl Default for GlobalOptions { } } +impl GlobalOptions { + /// Resolve conflicting options. + // TODO: https://github.com/clap-rs/clap/issues/815 + pub fn resolve_overrides(&mut self) { + if self.no_eval_cache { + self.eval_cache = false; + } + } +} + #[derive(Subcommand, Clone)] -pub(crate) enum Commands { +pub enum Commands { #[command(about = "Scaffold devenv.yaml, devenv.nix, .gitignore and .envrc.")] Init { target: Option, @@ -233,7 +270,7 @@ pub(crate) enum Commands { #[derive(Subcommand, Clone)] #[clap(about = "Start or stop processes. https://devenv.sh/processes/")] -pub(crate) enum ProcessesCommand { +pub enum ProcessesCommand { #[command(alias = "start", about = "Start processes in the foreground.")] Up { process: Option, @@ -249,7 +286,7 @@ pub(crate) enum ProcessesCommand { #[derive(Subcommand, Clone)] #[clap(about = "Run tasks. https://devenv.sh/tasks/")] -pub(crate) enum TasksCommand { +pub enum TasksCommand { #[command(about = "Run tasks.")] Run { tasks: Vec }, } @@ -259,7 +296,7 @@ pub(crate) enum TasksCommand { about = "Build, copy, or run a container. https://devenv.sh/containers/", arg_required_else_help(true) )] -pub(crate) enum ContainerCommand { +pub enum ContainerCommand { #[command(about = "Build a container.")] Build { name: String }, @@ -272,7 +309,7 @@ pub(crate) enum ContainerCommand { #[derive(Subcommand, Clone)] #[clap(about = "Add an input to devenv.yaml. https://devenv.sh/inputs/")] -pub(crate) enum InputsCommand { +pub enum InputsCommand { #[command(about = "Add an input to devenv.yaml.")] Add { #[arg(help = "The name of the input.")] diff --git a/devenv/src/cnix.rs b/devenv/src/cnix.rs index 96e2132c7..f7ecd6ca5 100644 --- a/devenv/src/cnix.rs +++ b/devenv/src/cnix.rs @@ -1,6 +1,7 @@ use crate::{cli, config, log}; use miette::{bail, IntoDiagnostic, Result, WrapErr}; use serde::Deserialize; +use sqlx::SqlitePool; use std::cell::{Ref, RefCell}; use std::collections::HashMap; use std::env; @@ -14,6 +15,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; pub struct Nix<'a> { logger: log::Logger, pub options: Options<'a>, + pool: SqlitePool, // TODO: all these shouldn't be here config: config::Config, global_options: cli::GlobalOptions, @@ -21,79 +23,109 @@ pub struct Nix<'a> { cachix_trusted_keys: PathBuf, devenv_home_gc: PathBuf, devenv_dot_gc: PathBuf, + devenv_dotfile: PathBuf, devenv_root: PathBuf, } #[derive(Clone)] pub struct Options<'a> { pub replace_shell: bool, + pub cache_output: bool, pub logging: bool, pub logging_stdout: bool, pub nix_flags: &'a [&'a str], } impl<'a> Nix<'a> { - pub fn new>( + pub async fn new>( logger: log::Logger, config: config::Config, global_options: cli::GlobalOptions, cachix_trusted_keys: P, devenv_home_gc: P, + devenv_dotfile: P, devenv_dot_gc: P, devenv_root: P, - ) -> Self { + ) -> Result { let cachix_trusted_keys = cachix_trusted_keys.as_ref().to_path_buf(); let devenv_home_gc = devenv_home_gc.as_ref().to_path_buf(); + let devenv_dotfile = devenv_dotfile.as_ref().to_path_buf(); let devenv_dot_gc = devenv_dot_gc.as_ref().to_path_buf(); let devenv_root = devenv_root.as_ref().to_path_buf(); let cachix_caches = RefCell::new(None); + let options = Options { + replace_shell: false, + // Individual commands opt into caching + cache_output: false, + logging: true, + logging_stdout: false, + nix_flags: &[ + "--show-trace", + "--extra-experimental-features", + "nix-command", + "--extra-experimental-features", + "flakes", + "--option", + "warn-dirty", + "false", + "--keep-going", + ], + }; + + let database_url = format!( + "sqlite:{}/nix-eval-cache.db", + devenv_dotfile.to_string_lossy() + ); + let pool = devenv_eval_cache::db::setup_db(database_url) + .await + .into_diagnostic()?; - Self { + Ok(Self { logger, - cachix_caches, + options, + pool, config, global_options, - options: Options { - replace_shell: false, - logging: true, - logging_stdout: false, - nix_flags: &[ - "--show-trace", - "--extra-experimental-features", - "nix-command", - "--extra-experimental-features", - "flakes", - "--option", - "warn-dirty", - "false", - "--keep-going", - ], - }, + cachix_caches, cachix_trusted_keys, devenv_home_gc, devenv_dot_gc, + devenv_dotfile, devenv_root, - } + }) } - pub async fn develop(&self, args: &[&str], replace_shell: bool) -> Result { + pub async fn develop( + &self, + args: &[&str], + replace_shell: bool, + ) -> Result { let options = Options { logging_stdout: true, + // Cannot cache this because we don't get the derivation back. + // We'd need to switch to print-dev-env and our own `nix develop`. + cache_output: false, replace_shell, ..self.options }; self.run_nix_with_substituters("nix", args, &options).await } - pub async fn dev_env(&self, json: bool, gc_root: &PathBuf) -> Result> { + pub async fn dev_env( + &self, + json: bool, + gc_root: &PathBuf, + ) -> Result { + let options = Options { + cache_output: true, + ..self.options + }; let gc_root_str = gc_root.to_str().expect("gc root should be utf-8"); let mut args: Vec<&str> = vec!["print-dev-env", "--profile", gc_root_str]; if json { args.push("--json"); } - - let options = Options { ..self.options }; let env = self .run_nix_with_substituters("nix", &args, &options) .await?; @@ -104,7 +136,7 @@ impl<'a> Nix<'a> { }; let args: Vec<&str> = vec!["-p", gc_root_str, "--delete-generations", "old"]; - self.run_nix("nix-env", &args, &options)?; + self.run_nix("nix-env", &args, &options).await?; let now_ns = get_now_with_nanoseconds(); let target = format!("{}-shell", now_ns); symlink_force( @@ -112,10 +144,11 @@ impl<'a> Nix<'a> { &fs::canonicalize(gc_root).expect("to resolve gc_root"), &self.devenv_home_gc.join(target), ); - Ok(env.stdout) + + Ok(env) } - pub fn add_gc(&self, name: &str, path: &Path) -> Result<()> { + pub async fn add_gc(&self, name: &str, path: &Path) -> Result<()> { self.run_nix( "nix-store", &[ @@ -125,7 +158,8 @@ impl<'a> Nix<'a> { path.to_str().unwrap(), ], &self.options, - )?; + ) + .await?; let link_path = self .devenv_dot_gc .join(format!("{}-{}", name, get_now_with_nanoseconds())); @@ -140,59 +174,76 @@ impl<'a> Nix<'a> { } pub async fn build(&self, attributes: &[&str]) -> Result> { - if !attributes.is_empty() { - // TODO: use eval underneath - let mut args: Vec = vec![ - "build".to_string(), - "--no-link".to_string(), - "--print-out-paths".to_string(), - ]; - args.extend(attributes.iter().map(|attr| format!(".#{}", attr))); - let args_str: Vec<&str> = args.iter().map(AsRef::as_ref).collect(); - let output = self - .run_nix_with_substituters("nix", &args_str, &self.options) - .await?; - Ok(String::from_utf8_lossy(&output.stdout) - .to_string() - .split_whitespace() - .map(|s| PathBuf::from(s.to_string())) - .collect()) - } else { - Ok(Vec::new()) + if attributes.is_empty() { + return Ok(Vec::new()); } + + let options = Options { + cache_output: true, + ..self.options + }; + // TODO: use eval underneath + let mut args: Vec = vec![ + "build".to_string(), + "--no-link".to_string(), + "--print-out-paths".to_string(), + ]; + args.extend(attributes.iter().map(|attr| format!(".#{}", attr))); + let args_str: Vec<&str> = args.iter().map(AsRef::as_ref).collect(); + let output = self + .run_nix_with_substituters("nix", &args_str, &options) + .await?; + Ok(String::from_utf8_lossy(&output.stdout) + .to_string() + .split_whitespace() + .map(|s| PathBuf::from(s.to_string())) + .collect()) } pub async fn eval(&self, attributes: &[&str]) -> Result { + let options = Options { + cache_output: true, + ..self.options + }; let mut args: Vec = vec!["eval", "--json"] .into_iter() .map(String::from) .collect(); args.extend(attributes.iter().map(|attr| format!(".#{}", attr))); let args = &args.iter().map(|s| s.as_str()).collect::>(); - let result = self.run_nix("nix", args, &self.options)?; + let result = self.run_nix("nix", args, &options).await?; String::from_utf8(result.stdout) .map_err(|err| miette::miette!("Failed to parse command output as UTF-8: {}", err)) } - pub fn update(&self, input_name: &Option) -> Result<()> { + pub async fn update(&self, input_name: &Option) -> Result<()> { match input_name { Some(input_name) => { self.run_nix( "nix", &["flake", "lock", "--update-input", input_name], &self.options, - )?; + ) + .await?; } None => { - self.run_nix("nix", &["flake", "update"], &self.options)?; + self.run_nix("nix", &["flake", "update"], &self.options) + .await?; } } Ok(()) } - pub fn metadata(&self) -> Result { + pub async fn metadata(&self) -> Result { + let options = Options { + cache_output: true, + ..self.options + }; + // TODO: use --json - let metadata = self.run_nix("nix", &["flake", "metadata"], &self.options)?; + let metadata = self + .run_nix("nix", &["flake", "metadata"], &options) + .await?; let re = regex::Regex::new(r"(Inputs:.+)$").unwrap(); let metadata_str = String::from_utf8_lossy(&metadata.stdout); @@ -201,7 +252,9 @@ impl<'a> Nix<'a> { None => "", }; - let info_ = self.run_nix("nix", &["eval", "--raw", ".#info"], &self.options)?; + let info_ = self + .run_nix("nix", &["eval", "--raw", ".#info"], &options) + .await?; Ok(format!( "{}\n{}", inputs, @@ -209,7 +262,7 @@ impl<'a> Nix<'a> { )) } - pub async fn search(&self, name: &str) -> Result { + pub async fn search(&self, name: &str) -> Result { self.run_nix_with_substituters( "nix", &["search", "--inputs-from", ".", "--json", "nixpkgs", name], @@ -234,14 +287,14 @@ impl<'a> Nix<'a> { } // Run Nix with debugger capability and return the output - pub fn run_nix( + pub async fn run_nix( &self, command: &str, args: &[&str], options: &Options<'a>, - ) -> Result { + ) -> Result { let cmd = self.prepare_command(command, args, options)?; - self.run_nix_command(cmd, options) + self.run_nix_command(cmd, options).await } pub async fn run_nix_with_substituters( @@ -249,18 +302,21 @@ impl<'a> Nix<'a> { command: &str, args: &[&str], options: &Options<'a>, - ) -> Result { + ) -> Result { let cmd = self .prepare_command_with_substituters(command, args, options) .await?; - self.run_nix_command(cmd, options) + self.run_nix_command(cmd, options).await } - fn run_nix_command( + async fn run_nix_command( &self, mut cmd: std::process::Command, options: &Options<'a>, - ) -> Result { + ) -> Result { + use devenv_eval_cache::internal_log::Verbosity; + use devenv_eval_cache::{supports_eval_caching, CachedCommand}; + let mut logger = self.logger.clone(); if !options.logging { @@ -279,46 +335,90 @@ impl<'a> Nix<'a> { display_command(&cmd), )); bail!("Failed to replace shell") - } else { - if options.logging { - cmd.stdin(process::Stdio::inherit()) - .stderr(process::Stdio::inherit()); - if options.logging_stdout { - cmd.stdout(std::process::Stdio::inherit()); - } + } + + if options.logging { + cmd.stdin(process::Stdio::inherit()) + .stderr(process::Stdio::inherit()); + if options.logging_stdout { + cmd.stdout(std::process::Stdio::inherit()); + } + } + + let result = if self.global_options.eval_cache + && options.cache_output + && supports_eval_caching(&cmd) + { + let mut cached_cmd = CachedCommand::new(&self.pool); + + cached_cmd.watch_path(self.devenv_root.join("devenv.yaml")); + + cached_cmd.unwatch_path(self.devenv_root.join(".devenv.flake.nix")); + // Ignore anything in .devenv. + cached_cmd.unwatch_path(&self.devenv_dotfile); + + if self.global_options.refresh_eval_cache { + cached_cmd.force_refresh(); } - let result = cmd + if options.logging { + let target_log_level = if self.global_options.verbose { + Verbosity::Talkative + } else if self.global_options.quiet { + Verbosity::Error + } else { + Verbosity::Info + }; + + cached_cmd.on_stderr(move |log| { + if let Some(msg) = log.get_log_msg_by_level(target_log_level) { + eprintln!("{msg}"); + } + }); + } + cached_cmd + .output(&mut cmd) + .await + .into_diagnostic() + .wrap_err_with(|| format!("Failed to run command `{}`", display_command(&cmd)))? + } else { + let output = cmd .output() .into_diagnostic() .wrap_err_with(|| format!("Failed to run command `{}`", display_command(&cmd)))?; + devenv_eval_cache::Output { + status: output.status, + stdout: output.stdout, + stderr: output.stderr, + paths: vec![], + } + }; - if !result.status.success() { - let code = match result.status.code() { - Some(code) => format!("with exit code {}", code), - None => "without exit code".to_string(), - }; - if options.logging { - eprintln!(); - self.logger.error(&format!( - "Command produced the following output:\n{}\n{}", - String::from_utf8_lossy(&result.stdout), - String::from_utf8_lossy(&result.stderr), - )); - } - if self.global_options.nix_debugger - && cmd.get_program().to_string_lossy().ends_with("bin/nix") - { - self.logger.info("Starting Nix debugger ..."); - cmd.arg("--debugger").exec(); - } - bail!(format!( - "Command `{}` failed with {code}", - display_command(&cmd) - )) - } else { - Ok(result) + if !result.status.success() { + let code = match result.status.code() { + Some(code) => format!("with exit code {}", code), + None => "without exit code".to_string(), + }; + if options.logging { + eprintln!(); + self.logger.error(&format!( + "Command produced the following output:\n{}\n{}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr), + )); + } + if self.global_options.nix_debugger + && cmd.get_program().to_string_lossy().ends_with("bin/nix") + { + self.logger.info("Starting Nix debugger ..."); + cmd.arg("--debugger").exec(); } + bail!(format!( + "Command `{}` failed with {code}", + display_command(&cmd) + )) + } else { + Ok(result) } } @@ -414,10 +514,10 @@ impl<'a> Nix<'a> { let max_jobs = self.global_options.max_jobs.to_string(); flags.push(&max_jobs); + // Disable the flake eval cache. flags.push("--option"); flags.push("eval-cache"); - let eval_cache = self.global_options.eval_cache.to_string(); - flags.push(&eval_cache); + flags.push("false"); // handle --nix-option key value for chunk in self.global_options.nix_option.chunks_exact(2) { @@ -517,7 +617,9 @@ impl<'a> Nix<'a> { } if !caches.caches.pull.is_empty() { - let store = self.run_nix("nix", &["store", "ping", "--json"], &no_logging)?; + let store = self + .run_nix("nix", &["store", "ping", "--json"], &no_logging) + .await?; let trusted = serde_json::from_slice::(&store.stdout) .expect("Failed to parse JSON") .trusted; diff --git a/devenv/src/devenv.rs b/devenv/src/devenv.rs index 4a33ced6d..eaa8f78b1 100644 --- a/devenv/src/devenv.rs +++ b/devenv/src/devenv.rs @@ -3,7 +3,7 @@ use clap::crate_version; use cli_table::Table; use cli_table::{print_stderr, WithTitle}; use include_dir::{include_dir, Dir}; -use miette::{bail, IntoDiagnostic, Result, WrapErr}; +use miette::{bail, Result}; use nix::sys::signal; use nix::unistd::Pid; use serde::Deserialize; @@ -34,32 +34,32 @@ pub struct DevenvOptions { } pub struct Devenv { - pub(crate) config: config::Config, - pub(crate) global_options: cli::GlobalOptions, + pub config: config::Config, + pub global_options: cli::GlobalOptions, - pub(crate) logger: log::Logger, - pub(crate) log_progress: log::LogProgressCreator, + logger: log::Logger, + log_progress: log::LogProgressCreator, nix: cnix::Nix<'static>, // All kinds of paths - xdg_dirs: xdg::BaseDirectories, - pub(crate) devenv_root: PathBuf, + devenv_root: PathBuf, devenv_dotfile: PathBuf, devenv_dot_gc: PathBuf, devenv_home_gc: PathBuf, devenv_tmp: String, devenv_runtime: PathBuf, - pub(crate) assembled: bool, - pub(crate) dirs_created: bool, - pub(crate) has_processes: Option, + assembled: bool, + has_processes: Option, - pub(crate) container_name: Option, + // TODO: make private. + // Pass as an arg or have a setter. + pub container_name: Option, } impl Devenv { - pub fn new(options: DevenvOptions) -> Self { + pub async fn new(options: DevenvOptions) -> Self { let xdg_dirs = xdg::BaseDirectories::with_prefix("devenv").unwrap(); let devenv_home = xdg_dirs.get_data_home(); let cachix_trusted_keys = devenv_home.join("cachix_trusted_keys.json"); @@ -102,22 +102,31 @@ impl Devenv { log::LogProgressCreator::Logging }; + xdg_dirs + .create_data_directory(Path::new("devenv")) + .expect("Failed to create DEVENV_HOME directory"); + std::fs::create_dir_all(&devenv_home_gc) + .expect("Failed to create DEVENV_HOME_GC directory"); + std::fs::create_dir_all(&devenv_dot_gc).expect("Failed to create .devenv/gc directory"); + let nix = cnix::Nix::new( logger.clone(), options.config.clone(), global_options.clone(), cachix_trusted_keys, devenv_home_gc.clone(), + devenv_dotfile.clone(), devenv_dot_gc.clone(), devenv_root.clone(), - ); + ) + .await + .unwrap(); // TODO: handle error Self { config: options.config, global_options, logger, log_progress, - xdg_dirs, devenv_root, devenv_dotfile, devenv_dot_gc, @@ -126,7 +135,6 @@ impl Devenv { devenv_runtime, nix, assembled: false, - dirs_created: false, has_processes: None, container_name: None, } @@ -200,10 +208,10 @@ impl Devenv { } pub async fn print_dev_env(&mut self, json: bool) -> Result<()> { - let (env, _) = self.get_dev_environment(json, false).await?; + let env = self.get_dev_environment(json, false).await?; print!( "{}", - String::from_utf8(env).expect("Failed to convert env to utf-8") + String::from_utf8(env.output).expect("Failed to convert env to utf-8") ); Ok(()) } @@ -231,11 +239,11 @@ impl Devenv { args: &[String], ) -> Result> { self.assemble(false)?; - let (_, gc_root) = self.get_dev_environment(false, true).await?; + let env = self.get_dev_environment(false, true).await?; let mut develop_args = vec![ "develop", - gc_root.to_str().expect("gc root should be utf-8"), + env.gc_root.to_str().expect("gc root should be utf-8"), ]; let default_clean = config::Clean { @@ -277,7 +285,7 @@ impl Devenv { Ok(develop_args.into_iter().map(|s| s.to_string()).collect()) } - pub fn update(&mut self, input_name: &Option) -> Result<()> { + pub async fn update(&mut self, input_name: &Option) -> Result<()> { let msg = match input_name { Some(input_name) => format!("Updating devenv.lock with input {input_name}"), None => "Updating devenv.lock".to_string(), @@ -285,7 +293,7 @@ impl Devenv { let _logprogress = self.log_progress.with_newline(&msg); self.assemble(false)?; - self.nix.update(input_name)?; + self.nix.update(input_name).await?; Ok(()) } @@ -573,9 +581,9 @@ impl Devenv { } } - pub fn info(&mut self) -> Result<()> { + pub async fn info(&mut self) -> Result<()> { self.assemble(false)?; - let output = self.nix.metadata()?; + let output = self.nix.metadata().await?; println!("{}", output); Ok(()) } @@ -640,7 +648,7 @@ impl Devenv { .to_str() .expect("Failed to get proc script path") .to_string(); - self.nix.add_gc("procfilescript", &proc_script[0])?; + self.nix.add_gc("procfilescript", &proc_script[0]).await?; } { let _logprogress = self.log_progress.with_newline("Starting processes"); @@ -744,23 +752,6 @@ impl Devenv { Ok(()) } - pub fn create_directories(&mut self) -> Result<()> { - if !self.dirs_created { - self.xdg_dirs - .create_data_directory(Path::new("devenv")) - .into_diagnostic() - .wrap_err("Failed to create DEVENV_HOME directory")?; - std::fs::create_dir_all(&self.devenv_home_gc) - .into_diagnostic() - .wrap_err("Failed to create DEVENV_HOME_GC directory")?; - std::fs::create_dir_all(&self.devenv_dot_gc) - .into_diagnostic() - .wrap_err("Failed to create .devenv/gc directory")?; - self.dirs_created = true; - } - Ok(()) - } - pub fn assemble(&mut self, is_testing: bool) -> Result<()> { if self.assembled { return Ok(()); @@ -799,6 +790,8 @@ impl Devenv { serde_json::to_string(&self.config).unwrap(), ) .expect("Failed to write devenv.json"); + // TODO: superceded by eval caching. + // Remove once direnvrc migration is implemented. fs::write( self.devenv_dotfile.join("imports.txt"), self.config.imports.join("\n"), @@ -838,11 +831,7 @@ impl Devenv { Ok(()) } - pub async fn get_dev_environment( - &mut self, - json: bool, - logging: bool, - ) -> Result<(Vec, PathBuf)> { + pub async fn get_dev_environment(&mut self, json: bool, logging: bool) -> Result { self.assemble(false)?; let _logprogress = if logging { Some(self.log_progress.with_newline("Building shell")) @@ -851,10 +840,29 @@ impl Devenv { }; let gc_root = self.devenv_dot_gc.join("shell"); let env = self.nix.dev_env(json, &gc_root).await?; - Ok((env, gc_root)) + + std::fs::write( + self.devenv_dotfile.join("input-paths.txt"), + env.paths + .iter() + .map(|fp| fp.path.to_string_lossy()) + .collect::>() + .join("\n"), + ) + .expect("Failed to write input-paths.txt"); + + Ok(DevEnv { + output: env.stdout, + gc_root, + }) } } +pub struct DevEnv { + output: Vec, + gc_root: PathBuf, +} + #[derive(Deserialize)] struct PackageResults(HashMap); diff --git a/devenv/src/lib.rs b/devenv/src/lib.rs index 1c78f0c0c..c95ab36ee 100644 --- a/devenv/src/lib.rs +++ b/devenv/src/lib.rs @@ -1,5 +1,5 @@ -mod cli; -pub mod cnix; +pub mod cli; +pub(crate) mod cnix; pub mod config; mod devenv; pub mod log; diff --git a/devenv/src/log.rs b/devenv/src/log.rs index 98f3e68d7..a1ca46212 100644 --- a/devenv/src/log.rs +++ b/devenv/src/log.rs @@ -99,7 +99,7 @@ impl Logger { self.log(message, Level::Warn); } - fn log(&self, message: &str, level: Level) { + pub fn log(&self, message: &str, level: Level) { if level > self.level { return; } diff --git a/devenv/src/main.rs b/devenv/src/main.rs index 25d36634a..a20c1ca40 100644 --- a/devenv/src/main.rs +++ b/devenv/src/main.rs @@ -1,18 +1,13 @@ -mod cli; -mod cnix; -mod config; -mod devenv; -mod log; -mod tasks; - -use clap::{crate_version, Parser}; -use cli::{Cli, Commands, ContainerCommand, InputsCommand, ProcessesCommand, TasksCommand}; -use devenv::Devenv; +use clap::crate_version; +use devenv::{ + cli::{Cli, Commands, ContainerCommand, InputsCommand, ProcessesCommand, TasksCommand}, + config, log, Devenv, +}; use miette::Result; #[tokio::main] async fn main() -> Result<()> { - let cli = Cli::parse(); + let cli = Cli::parse_and_resolve_options(); if let Commands::Version { .. } = cli.command { println!( @@ -61,8 +56,7 @@ async fn main() -> Result<()> { } } - let mut devenv = Devenv::new(options); - devenv.create_directories()?; + let mut devenv = Devenv::new(options).await; match cli.command { Commands::Shell { cmd, args } => devenv.shell(&cmd, &args, true).await, @@ -129,10 +123,10 @@ async fn main() -> Result<()> { Commands::Init { target } => devenv.init(&target), Commands::Search { name } => devenv.search(&name).await, Commands::Gc {} => devenv.gc(), - Commands::Info {} => devenv.info(), + Commands::Info {} => devenv.info().await, Commands::Repl {} => devenv.repl(), Commands::Build { attributes } => devenv.build(&attributes).await, - Commands::Update { name } => devenv.update(&name), + Commands::Update { name } => devenv.update(&name).await, Commands::Up { process, detach } => devenv.up(process.as_deref(), &detach, &detach).await, Commands::Processes { command } => match command { ProcessesCommand::Up { process, detach } => { diff --git a/direnvrc b/direnvrc index e9c8097ee..cb47ef057 100644 --- a/direnvrc +++ b/direnvrc @@ -115,8 +115,8 @@ use_devenv() { exit 0 fi - if [[ -f "$flake_dir/.devenv/imports.txt" ]]; then - for file in $(cat "$flake_dir/.devenv/imports.txt"); do + if [[ -f "$flake_dir/.devenv/input-paths.txt" ]]; then + for file in $(cat "$flake_dir/.devenv/input-paths.txt"); do files_to_watch+=("$file") done fi diff --git a/docs/blog/posts/devenv-v1.3-nix-caching.md b/docs/blog/posts/devenv-v1.3-nix-caching.md new file mode 100644 index 000000000..387bdfafc --- /dev/null +++ b/docs/blog/posts/devenv-v1.3-nix-caching.md @@ -0,0 +1,71 @@ +--- +draft: true +date: 2024-09-30 +authors: + - sandydoo +--- + +# devenv 1.3: Instant-er developer environments with caching + +Hot on the heels of the previous release, we're releasing devenv 1.3, which brings caching to Nix commands run inside of devenv. + +We've hooked up an SQLite-backed cache to automatically detect + +Once cached, the results of a Nix command can be recalled in single-digit milliseconds. +And if any of the automatically-detected inputs change, the cache is invalidated and the command is re-run. + +<> + +You can toggle the cache off with `--no-eval-cache`. +If you run into any issues, you can refresh the cache with `--refresh-eval-cache`. + +## How does it work? + +Behind the scenes, devenv now parses Nix's internal logs to determine which files and directories were accessed during evaluation. +This approach is very much inspired by [lorri](https://github.com/nix-community/lorri) and ... +The paths, the hash of their contents, and their last-modified timestamp are stored in a SQLite database. + +## Why not use the built-in flake evaluation cache? + +Since v2.4, Nix has had a built-in evaluation cache for flakes attributes, which is enabled by default. +It's worth pointing out that it's implemented at the command-line level; it is not a true expression cache. +This makes it quite limited in terms of what it can cache. +Calls to `getFlake` are not cached, and neither are any intermediate evaluation results. +Nonetheless, it has it's uses and it does significantly speed up repeated calls to `nix build`. + +Coming back to devenv, we made the choice to disable the built-in cache by default since v1.0. +We found that it had a tendency to aggresively cache evaluation errors, leading to a lot of frustration amongst our users. + +Running our own cache gives us more control and visibility over the caching process, and allows us to improve our integration with other tools, like direnv. + +The way devenv works, you can think of it as a composite command made of a series of `nix` commands. +Each individual call to `nix` is relatively expensive, but we can cache and reuse each step between devenv commands. +This makes our use of caching far more effective. +We can also predictably cache the outputs of a wider range of commands, like `nix eval` and `nix print-dev-env`. + +## What makes this different from lorri, or direnv and nix-direnv? + +Unlike `lorri`, devenv doesn't require a separate daemon running in the background. +Re-evaluation never happens in the background and can easily be aborted. + +`direnv`, and its sister project `nix-direnv`, are great for caching the evaluated Nix environment, but are limited in their ability to detect changes. +The best we could previously do was manually watch the files we reasonably expected to affect evaluation. + +On the other hand, `devenv`'s new caching layer can automatically detect dependencies from path types, `import`s, and even `builtins.readFile` and `builtins.readDir`. + +## What's next? + +### More caching + +`nix develop` currently remains a rather slow and uncachable pain point. + +### Better options for tracking files + +One of the challenges we face in adapting Nix to real-world developer environments is in how it treats file paths. +All paths are first copied to the store and then replaced with a store path. +For our use-case, this can be slow, unnecessary, and, in some cases, even a security risk. +Files that impact the instantiation of the shell environment but dont affect the Nix evaluation are a common occurance — think configurations files, or package manager lock files. +There's no need to copy them to the store, but we'd like to know they're there to reload the environment when they change. +We're working on a solution that leverages our new caching infrastructure to provide a transparent way to track these files. + +Sander diff --git a/package.nix b/package.nix index c5990b0df..23bdb4be5 100644 --- a/package.nix +++ b/package.nix @@ -11,12 +11,16 @@ pkgs.rustPlatform.buildRustPackage { ".*Cargo\.toml" ".*Cargo\.lock" ".*devenv(/.*)?" - ".*tasks(/.*)?" + ".*devenv-eval-cache(/.*)?" ".*devenv-run-tests(/.*)?" ".*xtask(/.*)?" + ".*tasks(/.*)?" ]; - cargoBuildFlags = if build_tasks then [ "-p tasks" ] else [ "-p devenv -p devenv-run-tests" ]; + cargoBuildFlags = + if build_tasks + then [ "-p tasks" ] + else [ "-p devenv -p devenv-run-tests" ]; doCheck = !build_tasks; @@ -28,12 +32,23 @@ pkgs.rustPlatform.buildRustPackage { pkgs.makeWrapper pkgs.pkg-config pkgs.installShellFiles + pkgs.sqlx-cli ]; buildInputs = [ pkgs.openssl ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ pkgs.darwin.apple_sdk.frameworks.SystemConfiguration ]; + # Force sqlx to use the prepared queries + SQLX_OFFLINE = true; + # A local database to use for preparing queries + DATABASE_URL = "sqlite:nix-eval-cache.db"; + + preBuild = '' + cargo sqlx database setup --source devenv-eval-cache/migrations + cargo sqlx prepare --workspace + ''; + postInstall = pkgs.lib.optionalString (!build_tasks) '' wrapProgram $out/bin/devenv \ --set DEVENV_NIX ${inputs.nix.packages.${pkgs.stdenv.system}.nix} \ diff --git a/tasks/Cargo.toml b/tasks/Cargo.toml index 69472260a..5bf36a924 100644 --- a/tasks/Cargo.toml +++ b/tasks/Cargo.toml @@ -6,6 +6,6 @@ license.workspace = true [dependencies] clap.workspace = true -devenv = { path = "../devenv" } +devenv.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 38d0ac42a..f6712f611 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -8,5 +8,5 @@ license.workspace = true clap.workspace = true clap_complete = "4.5.7" clap_mangen = "0.2.22" -devenv = { path = "../devenv" } +devenv.workspace = true miette.workspace = true diff --git a/xtask/src/cli.rs b/xtask/src/cli.rs deleted file mode 100644 index 741f7c14e..000000000 --- a/xtask/src/cli.rs +++ /dev/null @@ -1 +0,0 @@ -include!("../../devenv/src/cli.rs"); diff --git a/xtask/src/lib.rs b/xtask/src/lib.rs index 4931e8a47..5c2d71a4a 100644 --- a/xtask/src/lib.rs +++ b/xtask/src/lib.rs @@ -1,3 +1,2 @@ -pub(crate) mod cli; pub mod manpage; pub mod shell_completion; diff --git a/xtask/src/manpage.rs b/xtask/src/manpage.rs index a2dd80ff8..0f190b0b5 100644 --- a/xtask/src/manpage.rs +++ b/xtask/src/manpage.rs @@ -1,5 +1,5 @@ -use super::cli::Cli; use clap::CommandFactory; +use devenv::cli::Cli; use miette::{IntoDiagnostic, Result}; use std::fs; use std::path::{Path, PathBuf}; diff --git a/xtask/src/shell_completion.rs b/xtask/src/shell_completion.rs index 70036561a..dc2eaa784 100644 --- a/xtask/src/shell_completion.rs +++ b/xtask/src/shell_completion.rs @@ -1,5 +1,5 @@ -use crate::cli::Cli; use clap::CommandFactory; +use devenv::cli::Cli; use miette::{IntoDiagnostic, Result}; use std::fs; use std::path::{Path, PathBuf};