From 8b09936d7c8608c0e58f9d991e257ffd596e698c Mon Sep 17 00:00:00 2001 From: Patryk Wychowaniec Date: Fri, 14 Jul 2023 11:21:27 +0200 Subject: [PATCH 1/5] Use TOML serializer from nixpkgs Closes https://github.com/nix-community/naersk/issues/263 --- build.nix | 7 +- builtins/default.nix | 4 +- builtins/to-toml.nix | 139 --------------------------------- default.nix | 5 +- lib.nix | 9 +-- nix/sources.json | 30 ++++--- test/default.nix | 22 +++++- test/slow/default.nix | 1 + test/slow/probe-rs/default.nix | 33 ++++++++ 9 files changed, 87 insertions(+), 163 deletions(-) delete mode 100644 builtins/to-toml.nix create mode 100644 test/slow/probe-rs/default.nix diff --git a/build.nix b/build.nix index 8ab8d97..750ac1e 100644 --- a/build.nix +++ b/build.nix @@ -62,6 +62,7 @@ , writeText , runCommandLocal , remarshal +, formats , crateDependencies , zstd , fetchurl @@ -72,7 +73,7 @@ let builtinz = builtins // import ./builtins - { inherit lib writeText remarshal runCommandLocal; }; + { inherit lib writeText remarshal runCommandLocal formats; }; drvAttrs = { name = "${pname}-${version}"; @@ -87,7 +88,7 @@ let # The cargo config with source replacement. Replaces both crates.io crates # and git dependencies. - cargoconfig = builtinz.toTOML { + cargoconfig = builtinz.writeTOML "config" { source = { crates-io = { replace-with = "nix-sources"; }; nix-sources = { @@ -235,7 +236,7 @@ let export CARGO_HOME=''${CARGO_HOME:-$PWD/.cargo-home} mkdir -p $CARGO_HOME - echo "$cargoconfig" > $CARGO_HOME/config + cp "$cargoconfig" $CARGO_HOME/config runHook postConfigure ''; diff --git a/builtins/default.nix b/builtins/default.nix index 755647a..b1cf2fa 100644 --- a/builtins/default.nix +++ b/builtins/default.nix @@ -3,12 +3,12 @@ , writeText , runCommandLocal , remarshal +, formats }: rec { - toTOML = import ./to-toml.nix { inherit lib; }; - writeTOML = name: attrs: writeText name (toTOML attrs); + writeTOML = (formats.toml { }).generate; readTOML = usePure: f: if usePure then diff --git a/builtins/to-toml.nix b/builtins/to-toml.nix deleted file mode 100644 index 407569e..0000000 --- a/builtins/to-toml.nix +++ /dev/null @@ -1,139 +0,0 @@ -{ lib }: -let - inherit (lib) - length - elemAt - concatMap - all - concatLists - concatStringsSep - concatMapStringsSep - mapAttrsToList - ; - - inherit (builtins) - abort - match - toJSON - typeOf - ; - - quoteKey = k: - if match "[a-zA-Z]+" k == [] - then k - else quoteString k; - - quoteString = builtins.toJSON; - - outputValInner = v: - let - ty = tomlTy v; - in - if ty == "set" then - let - vals = mapAttrsToList - (k': v': "${quoteKey k'} = ${outputValInner v'}") v; - valsStr = concatStringsSep ", " vals; - in - "{ ${valsStr} }" else - outputVal v; - - outputVal = v: - let - ty = tomlTy v; - in - if (ty == "bool" || ty == "int") then - builtins.toJSON v - else - if ty == "string" then - quoteString v - else - if ty == "list" || ty == "list_of_attrs" then - let - vals = map quoteString v; - valsStr = concatStringsSep ", " vals; - in - "[ ${valsStr} ]" - else - if ty == "set" then - abort "unsupported set for not-inner value" - else abort "Not implemented: type ${ty}"; - - outputKeyValInner = k: v: - let - ty = tomlTy v; - in - if ty == "set" then - let - vals = mapAttrsToList - (k': v': "${quoteKey k'} = ${outputValInner v'}") v; - valsStr = concatStringsSep ", " vals; - in - [ "${quoteKey k} = { ${valsStr} }" ] else - outputKeyVal k v; - - # Returns a list of strings; one string per line - outputKeyVal = k: v: - let - ty = tomlTy v; - in - if ty == "bool" || ty == "int" then - [ "${quoteKey k} = ${outputValInner v}" ] - else - if ty == "string" then - [ "${quoteKey k} = ${quoteString v}" ] - else - if ty == "list_of_attrs" then - concatMap ( - inner: - [ "[[${k}]]" ] ++ (concatLists (mapAttrsToList outputKeyValInner inner)) - ) v - else - if ty == "list" then - let - vals = map quoteString v; - valsStr = concatStringsSep ", " vals; - in - [ "${quoteKey k} = [ ${valsStr} ]" ] else - if ty == "set" then - [ "[${k}]" ] ++ (concatLists (mapAttrsToList outputKeyValInner v)) - else abort "Not implemented: type ${ty} for key ${k}"; - - tomlTy = x: - if typeOf x == "string" then "string" else - if typeOf x == "bool" then "bool" else - if typeOf x == "int" then "int" else - if typeOf x == "float" then "float" else - if typeOf x == "set" then - if lib.isDerivation x then "string" else "set" else - if typeOf x == "list" then - if length x == 0 then "list" - else - let - ty = typeOf (elemAt x 0); - in - #assert (all (v: typeOf v == ty) x); - if ty == "set" then "list_of_attrs" else "list" - else abort "Not implemented: toml type for ${typeOf x}"; - - toTOML = attrs: - assert (typeOf attrs == "set"); - let - byTy = lib.foldl - ( - acc: x: - let - ty = tomlTy x.v; - in - acc // { "${ty}" = (acc.${ty} or []) ++ [ x ]; } - ) - {} (mapAttrsToList (k: v: { inherit k v; }) attrs); - in - concatMapStringsSep "\n" - (kv: concatStringsSep "\n" (outputKeyVal kv.k kv.v)) - ( - (byTy.string or []) ++ (byTy.int or []) ++ (byTy.float or []) ++ (byTy.list or []) ++ (byTy.list_of_attrs or []) ++ (byTy.set or []) - ) - ; -in -toTOML diff --git a/default.nix b/default.nix index 35de254..61b44ec 100644 --- a/default.nix +++ b/default.nix @@ -5,6 +5,7 @@ , lib , lndir , remarshal +, formats , rsync , runCommandLocal , rustc @@ -15,10 +16,10 @@ }@defaultBuildAttrs: let - libb = import ./lib.nix { inherit lib writeText runCommandLocal remarshal; }; + libb = import ./lib.nix { inherit lib writeText runCommandLocal remarshal formats; }; builtinz = builtins // import ./builtins - { inherit lib writeText remarshal runCommandLocal; }; + { inherit lib writeText remarshal runCommandLocal formats; }; mkConfig = arg: import ./config.nix { inherit lib arg libb builtinz; }; diff --git a/lib.nix b/lib.nix index a615bf2..33a5875 100644 --- a/lib.nix +++ b/lib.nix @@ -1,8 +1,8 @@ -{ lib, writeText, runCommandLocal, remarshal }: +{ lib, writeText, runCommandLocal, remarshal, formats }: let builtinz = builtins // import ./builtins - { inherit lib writeText remarshal runCommandLocal; }; + { inherit lib writeText remarshal runCommandLocal formats; }; in rec { @@ -121,14 +121,13 @@ rec # A very minimal 'src' which makes cargo happy nonetheless dummySrc = - { cargoconfig # string + { cargoconfig # path , cargotomls # list , cargolock # attrset , copySources # list of paths that should be copied to the output , copySourcesFrom # path from which to copy ${copySources} }: let - config = writeText "config" cargoconfig; cargolock' = builtinz.writeTOML "Cargo.lock" cargolock; fixupCargoToml = cargotoml: @@ -154,7 +153,7 @@ rec { inherit copySources copySourcesFrom cargotomlss; } '' mkdir -p $out/.cargo - ${lib.optionalString (! isNull cargoconfig) "cp ${config} $out/.cargo/config"} + ${lib.optionalString (! isNull cargoconfig) "cp ${cargoconfig} $out/.cargo/config"} cp ${cargolock'} $out/Cargo.lock for tuple in $cargotomlss; do diff --git a/nix/sources.json b/nix/sources.json index ecf2b7d..0fa6a3e 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -1,14 +1,14 @@ { "agent-rs": { - "branch": "paulliu/add-cargo-lock", + "branch": "main", "description": "A collection of libraries and tools for building software around the Internet Computer, in Rust.", - "homepage": "https://sdk.dfinity.org/", - "owner": "ninegua", + "homepage": "", + "owner": "dfinity", "repo": "agent-rs", - "rev": "4a22e590516bc79ec3c75a320f7941e7762ea098", - "sha256": "0sacddc34nlfgldqghlwchgzjki177h5dsgpmdv70cm8hfy0sg7l", + "rev": "6d923cbc918852ef5bfaeb08e63c86580aa80ffe", + "sha256": "12y28nmv9af8sj55s998wydy08h32gvxr9dbhfxhfilxdairh0a2", "type": "tarball", - "url": "https://github.com/ninegua/agent-rs/archive/4a22e590516bc79ec3c75a320f7941e7762ea098.tar.gz", + "url": "https://github.com/dfinity/agent-rs/archive/6d923cbc918852ef5bfaeb08e63c86580aa80ffe.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "fenix": { @@ -65,10 +65,10 @@ "homepage": null, "owner": "NixOS", "repo": "nixpkgs", - "rev": "6c2318e451fc0eaf338fb461d9bfcc99869de758", - "sha256": "1djwkvjnn26v0xw1swwh7dgmi0yl0b1kb8kansrdwk0jhhqbl7zq", + "rev": "1df733d83081fe79c109b066c90badece6b8d8b1", + "sha256": "1m3s4xv35jilrmbv8hj9gzpbcqgb8vb0iqhzcm1261x8b1hiylvj", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/6c2318e451fc0eaf338fb461d9bfcc99869de758.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/1df733d83081fe79c109b066c90badece6b8d8b1.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs-21.05": { @@ -119,6 +119,18 @@ "url": "https://github.com/nushell/nushell/archive/85bfdca578157072e51e6972d370cfe63b0fda77.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, + "probe-rs": { + "branch": "master", + "description": "A debugging toolset and library for debugging embedded ARM and RISC-V targets on a separate host", + "homepage": "https://probe.rs", + "owner": "probe-rs", + "repo": "probe-rs", + "rev": "51fa324aef9f7c413988a3d18052b1bbc278a4c5", + "sha256": "0zb9s80hsz83ngjngs9cllp7gf8xq9jz0m3lwdhf08x3cp3bj6fd", + "type": "tarball", + "url": "https://github.com/probe-rs/probe-rs/archive/51fa324aef9f7c413988a3d18052b1bbc278a4c5.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "ripgrep-all": { "branch": "master", "description": "rga: ripgrep, but also search in PDFs, E-Books, Office documents, zip, tar.gz, etc.", diff --git a/test/default.nix b/test/default.nix index 75097f6..ede5b1c 100644 --- a/test/default.nix +++ b/test/default.nix @@ -2,9 +2,25 @@ let sources = import ../nix/sources.nix; - pkgs = import ../nix { - inherit system nixpkgs; - }; + pkgs = + let + pkgs' = import ../nix { + inherit system nixpkgs; + }; + + older-pkgs = import ../nix { + inherit system; + + nixpkgs = "nixpkgs-21.05"; + }; + + in + pkgs' // { + # Some of our tests use dynamically-built Git repositories that fail extra + # security checks introduced in newer Git versions - so for the time being + # let's pin our test-Git to an older version. + git = older-pkgs.git; + }; naersk = pkgs.callPackage ../default.nix { inherit (pkgs.rustPackages) cargo rustc; diff --git a/test/slow/default.nix b/test/slow/default.nix index d58dff1..81151d3 100644 --- a/test/slow/default.nix +++ b/test/slow/default.nix @@ -2,6 +2,7 @@ args: { agent-rs = import ./agent-rs args; lorri = import ./lorri args; nushell = import ./nushell args; + probe-rs = import ./probe-rs args; ripgrep-all = import ./ripgrep-all args; rustlings = import ./rustlings args; talent-plan = import ./talent-plan args; diff --git a/test/slow/probe-rs/default.nix b/test/slow/probe-rs/default.nix new file mode 100644 index 0000000..64f8310 --- /dev/null +++ b/test/slow/probe-rs/default.nix @@ -0,0 +1,33 @@ +{ sources, pkgs, ... }: +let + fenix = import sources.fenix { }; + + toolchain = (fenix.toolchainOf { + channel = "nightly"; + date = "2023-07-01"; + sha256 = "sha256-pWd4tAHP4QWGC3CKWZzDjzYANxATC6CGRmKuP2ZZv5k="; + }).toolchain; + + naersk = pkgs.callPackage ../../../default.nix { + cargo = toolchain; + rustc = toolchain; + }; + + app = naersk.buildPackage { + src = sources.probe-rs; + + buildInputs = with pkgs; [ + pkg-config + libusb1 + openssl + ] ++ lib.optionals stdenv.isDarwin [ + darwin.DarwinTools + darwin.apple_sdk.frameworks.AppKit + ]; + }; + +in +pkgs.runCommand "probe-rs-test" +{ + buildInputs = [ app ]; +} "rtthost --help && touch $out" From 51f67d7e3d4a1089f5fdbc5f2c43e822d9f2a9fd Mon Sep 17 00:00:00 2001 From: Patryk Wychowaniec Date: Sun, 30 Jul 2023 14:48:47 +0200 Subject: [PATCH 2/5] Use custom writeTOML on older nixpkgs --- .github/workflows/fast.yml | 2 +- builtins/default.nix | 42 +++++++++- builtins/to-toml.nix | 139 +++++++++++++++++++++++++++++++++ nix/sources.json | 30 ++++--- test/slow/agent-rs/default.nix | 15 +++- test/slow/probe-rs/default.nix | 15 +++- 6 files changed, 227 insertions(+), 16 deletions(-) create mode 100644 builtins/to-toml.nix diff --git a/.github/workflows/fast.yml b/.github/workflows/fast.yml index 6e340ae..64902f6 100644 --- a/.github/workflows/fast.yml +++ b/.github/workflows/fast.yml @@ -10,7 +10,7 @@ jobs: linux: strategy: matrix: - nixpkgs: [ nixpkgs, nixpkgs-21.05, nixpkgs-21.11 ] + nixpkgs: [ nixpkgs, nixpkgs-21.05, nixpkgs-22.05, nixpkgs-23.05 ] runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 diff --git a/builtins/default.nix b/builtins/default.nix index b1cf2fa..7bcf40e 100644 --- a/builtins/default.nix +++ b/builtins/default.nix @@ -8,7 +8,47 @@ rec { - writeTOML = (formats.toml { }).generate; + # Serializes given attrset into a TOML file. + # + # Usage: + # writeTOML path attrset + # + # On newer nixpkgs, this function invokes `lib.formats.toml` that nowadays + # handles all TOML documents properly. + # + # On older nixpkgs, where that serializer doesn't work correctly¹, we rely on + # a custom implementation (with its own tiny shortcomings²). + # + # TODO remove our custom serializer after nixpkgs v23 becomes more widely + # adopted + # + # ¹ e.g. cases like `[targets."cfg(\"something\")"]` are translated badly + # ² https://github.com/nix-community/naersk/issues/263 + writeTOML = + let + our-impl = + let + to-toml = import ./to-toml.nix { + inherit lib; + }; + + in + name: value: + runCommandLocal name { + value = to-toml value; + passAsFile = [ "value" ]; + } '' + cp "$valuePath" "$out" + cat "$out" + ''; + + nixpkgs-impl = (formats.toml { }).generate; + + in + if builtins.compareVersions lib.version "22.11" <= 0 then + our-impl + else + nixpkgs-impl; readTOML = usePure: f: if usePure then diff --git a/builtins/to-toml.nix b/builtins/to-toml.nix new file mode 100644 index 0000000..407569e --- /dev/null +++ b/builtins/to-toml.nix @@ -0,0 +1,139 @@ +{ lib }: +let + inherit (lib) + length + elemAt + concatMap + all + concatLists + concatStringsSep + concatMapStringsSep + mapAttrsToList + ; + + inherit (builtins) + abort + match + toJSON + typeOf + ; + + quoteKey = k: + if match "[a-zA-Z]+" k == [] + then k + else quoteString k; + + quoteString = builtins.toJSON; + + outputValInner = v: + let + ty = tomlTy v; + in + if ty == "set" then + let + vals = mapAttrsToList + (k': v': "${quoteKey k'} = ${outputValInner v'}") v; + valsStr = concatStringsSep ", " vals; + in + "{ ${valsStr} }" else + outputVal v; + + outputVal = v: + let + ty = tomlTy v; + in + if (ty == "bool" || ty == "int") then + builtins.toJSON v + else + if ty == "string" then + quoteString v + else + if ty == "list" || ty == "list_of_attrs" then + let + vals = map quoteString v; + valsStr = concatStringsSep ", " vals; + in + "[ ${valsStr} ]" + else + if ty == "set" then + abort "unsupported set for not-inner value" + else abort "Not implemented: type ${ty}"; + + outputKeyValInner = k: v: + let + ty = tomlTy v; + in + if ty == "set" then + let + vals = mapAttrsToList + (k': v': "${quoteKey k'} = ${outputValInner v'}") v; + valsStr = concatStringsSep ", " vals; + in + [ "${quoteKey k} = { ${valsStr} }" ] else + outputKeyVal k v; + + # Returns a list of strings; one string per line + outputKeyVal = k: v: + let + ty = tomlTy v; + in + if ty == "bool" || ty == "int" then + [ "${quoteKey k} = ${outputValInner v}" ] + else + if ty == "string" then + [ "${quoteKey k} = ${quoteString v}" ] + else + if ty == "list_of_attrs" then + concatMap ( + inner: + [ "[[${k}]]" ] ++ (concatLists (mapAttrsToList outputKeyValInner inner)) + ) v + else + if ty == "list" then + let + vals = map quoteString v; + valsStr = concatStringsSep ", " vals; + in + [ "${quoteKey k} = [ ${valsStr} ]" ] else + if ty == "set" then + [ "[${k}]" ] ++ (concatLists (mapAttrsToList outputKeyValInner v)) + else abort "Not implemented: type ${ty} for key ${k}"; + + tomlTy = x: + if typeOf x == "string" then "string" else + if typeOf x == "bool" then "bool" else + if typeOf x == "int" then "int" else + if typeOf x == "float" then "float" else + if typeOf x == "set" then + if lib.isDerivation x then "string" else "set" else + if typeOf x == "list" then + if length x == 0 then "list" + else + let + ty = typeOf (elemAt x 0); + in + #assert (all (v: typeOf v == ty) x); + if ty == "set" then "list_of_attrs" else "list" + else abort "Not implemented: toml type for ${typeOf x}"; + + toTOML = attrs: + assert (typeOf attrs == "set"); + let + byTy = lib.foldl + ( + acc: x: + let + ty = tomlTy x.v; + in + acc // { "${ty}" = (acc.${ty} or []) ++ [ x ]; } + ) + {} (mapAttrsToList (k: v: { inherit k v; }) attrs); + in + concatMapStringsSep "\n" + (kv: concatStringsSep "\n" (outputKeyVal kv.k kv.v)) + ( + (byTy.string or []) ++ (byTy.int or []) ++ (byTy.float or []) ++ (byTy.list or []) ++ (byTy.list_of_attrs or []) ++ (byTy.set or []) + ) + ; +in +toTOML diff --git a/nix/sources.json b/nix/sources.json index 0fa6a3e..12ea476 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -60,15 +60,15 @@ "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs": { - "branch": "master", + "branch": "nixpkgs-unstable", "description": "Nixpkgs/NixOS branches that track the Nixpkgs/NixOS channels", "homepage": null, "owner": "NixOS", "repo": "nixpkgs", - "rev": "1df733d83081fe79c109b066c90badece6b8d8b1", - "sha256": "1m3s4xv35jilrmbv8hj9gzpbcqgb8vb0iqhzcm1261x8b1hiylvj", + "rev": "d2b52322f35597c62abf56de91b0236746b2a03d", + "sha256": "1zrqry4m22rbscazxbcqf5ym7ckm6v7pvxpsml23whc1gjf2hkk3", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/1df733d83081fe79c109b066c90badece6b8d8b1.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/d2b52322f35597c62abf56de91b0236746b2a03d.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs-21.05": { @@ -83,16 +83,28 @@ "url": "https://github.com/NixOS/nixpkgs/archive/6d684ea3adef590a2174f2723134e1ea377272d2.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, - "nixpkgs-21.11": { - "branch": "release-21.11", + "nixpkgs-22.05": { + "branch": "release-22.05", + "description": "Nix Packages collection", + "homepage": "", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "380be19fbd2d9079f677978361792cb25e8a3635", + "sha256": "154x9swf494mqwi4z8nbq2f0sp8pwp4fvx51lqzindjfbb9yxxv5", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/380be19fbd2d9079f677978361792cb25e8a3635.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs-23.05": { + "branch": "release-23.05", "description": "Nix Packages collection", "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "432864f33c84af53192d4b23ee5871697e57f094", - "sha256": "0sqmasma9qin7nhfzv50d1la1mi1yazn7gx2lrxi9j61i9j8l3zm", + "rev": "b81af66deb21f73a70c67e5ea189568af53b1e8c", + "sha256": "172i5kzn8qja3k0rz9wfj86fgbw2ivc68yw3ra8g86byqc1rcw1m", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/432864f33c84af53192d4b23ee5871697e57f094.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/b81af66deb21f73a70c67e5ea189568af53b1e8c.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs-mozilla": { diff --git a/test/slow/agent-rs/default.nix b/test/slow/agent-rs/default.nix index 1b8452c..5a56ee7 100644 --- a/test/slow/agent-rs/default.nix +++ b/test/slow/agent-rs/default.nix @@ -1,5 +1,18 @@ -{ sources, naersk, pkgs, ... }: +{ sources, pkgs, ... }: +let + fenix = import sources.fenix { }; + toolchain = fenix.fromToolchainFile { + file = "${sources.agent-rs}/rust-toolchain.toml"; + sha256 = "sha256-DzNEaW724O8/B8844tt5AVHmSjSQ3cmzlU4BP90oRlY="; + }; + + naersk = pkgs.callPackage ../../../default.nix { + cargo = toolchain; + rustc = toolchain; + }; + +in naersk.buildPackage { src = sources.agent-rs; diff --git a/test/slow/probe-rs/default.nix b/test/slow/probe-rs/default.nix index 64f8310..62b4677 100644 --- a/test/slow/probe-rs/default.nix +++ b/test/slow/probe-rs/default.nix @@ -27,7 +27,14 @@ let }; in -pkgs.runCommand "probe-rs-test" -{ - buildInputs = [ app ]; -} "rtthost --help && touch $out" +if builtins.compareVersions pkgs.lib.version "22.11" <= 0 then + # Executing this test requires nixpkgs > 22.11 due to changes to the TOML + # serialization function. + # + # See `writeTOML` in this repository for more details. + true +else + pkgs.runCommand "probe-rs-test" + { + buildInputs = [ app ]; + } "rtthost --help && touch $out" From f15d348c92c002a41115c4ecc5301b83db562d24 Mon Sep 17 00:00:00 2001 From: Patryk Wychowaniec Date: Sun, 30 Jul 2023 14:57:18 +0200 Subject: [PATCH 3/5] ci: Bump --- .github/workflows/fast.yml | 12 ++++++------ .github/workflows/test.yml | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/fast.yml b/.github/workflows/fast.yml index 64902f6..3375af1 100644 --- a/.github/workflows/fast.yml +++ b/.github/workflows/fast.yml @@ -11,10 +11,10 @@ jobs: strategy: matrix: nixpkgs: [ nixpkgs, nixpkgs-21.05, nixpkgs-22.05, nixpkgs-23.05 ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v16 + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v22 with: CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} - name: "Run tests" @@ -22,10 +22,10 @@ jobs: env: nixpkgs: ${{ matrix.nixpkgs }} darwin: - runs-on: macos-10.15 + runs-on: macos-latest steps: - - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v16 + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v22 with: CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} - name: "Run tests" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a2d7acd..f345b16 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,11 +8,11 @@ jobs: linux: strategy: matrix: - nixpkgs: [ nixpkgs, nixpkgs-21.05, nixpkgs-21.11 ] - runs-on: ubuntu-20.04 + nixpkgs: [ nixpkgs, nixpkgs-21.05, nixpkgs-22.05, nixpkgs-23.05 ] + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v16 + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v22 with: CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} - name: "Run tests" @@ -20,10 +20,10 @@ jobs: env: nixpkgs: ${{ matrix.nixpkgs }} darwin: - runs-on: macos-10.15 + runs-on: macos-latest steps: - - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v16 + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v22 with: CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} - name: "Run tests" From 704df26f1dc62e56dd8d1335bef46b4f1ea7af76 Mon Sep 17 00:00:00 2001 From: Patryk Wychowaniec Date: Tue, 15 Aug 2023 10:22:24 +0200 Subject: [PATCH 4/5] code review: Pin nixpkgs to master --- nix/sources.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nix/sources.json b/nix/sources.json index 12ea476..d2a14ef 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -60,15 +60,15 @@ "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs": { - "branch": "nixpkgs-unstable", + "branch": "master", "description": "Nixpkgs/NixOS branches that track the Nixpkgs/NixOS channels", "homepage": null, "owner": "NixOS", "repo": "nixpkgs", - "rev": "d2b52322f35597c62abf56de91b0236746b2a03d", - "sha256": "1zrqry4m22rbscazxbcqf5ym7ckm6v7pvxpsml23whc1gjf2hkk3", + "rev": "67bcf01c471217a9a1ac7e8aac3e5dde182ed6e9", + "sha256": "1wpmq4svsvb1scn5fmm5m0pq4wrdq4chn569x2v7qdfbqjir4sd0", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/d2b52322f35597c62abf56de91b0236746b2a03d.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/67bcf01c471217a9a1ac7e8aac3e5dde182ed6e9.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs-21.05": { From 05b6ebef78eca6608061455e6128070427f62b91 Mon Sep 17 00:00:00 2001 From: Patryk Wychowaniec Date: Tue, 15 Aug 2023 13:29:15 +0200 Subject: [PATCH 5/5] fix: Use separate sources for crates.io & Git deps --- build.nix | 56 +++++++++++++++++++++++++++++++----------------------- config.nix | 2 +- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/build.nix b/build.nix index 750ac1e..aeff784 100644 --- a/build.nix +++ b/build.nix @@ -63,7 +63,7 @@ , runCommandLocal , remarshal , formats -, crateDependencies +, cratesIoDependencies , zstd , fetchurl , lndir @@ -84,15 +84,18 @@ let postInstall ; - crate_sources = unpackedDependencies; + cratesio_sources = unpackedCratesIoDependencies; + git_sources = unpackedGitDependencies; # The cargo config with source replacement. Replaces both crates.io crates # and git dependencies. cargoconfig = builtinz.writeTOML "config" { source = { - crates-io = { replace-with = "nix-sources"; }; - nix-sources = { - directory = unpackedDependencies; + crates-io = { + directory = unpackedCratesIoDependencies; + }; + git = { + directory = unpackedGitDependencies; }; } // lib.listToAttrs ( map @@ -108,7 +111,7 @@ let name = "${e.url}${key}"; value = lib.filterAttrs (n: _: n == "rev" || n == "tag" || n == "branch") e // { git = e.url; - replace-with = "nix-sources"; + replace-with = "git"; }; } ) @@ -188,7 +191,6 @@ let log "RUST_TEST_THREADS: $RUST_TEST_THREADS" log "cargo_bins_jq_filter: $cargo_bins_jq_filter" log "cargo_build_output_json (created): $cargo_build_output_json" - log "crate_sources: $crate_sources" log "RUSTFLAGS: $RUSTFLAGS" log "CARGO_BUILD_RUSTFLAGS: $CARGO_BUILD_RUSTFLAGS" @@ -196,13 +198,18 @@ let # Remove the source path(s) in Rust if [ -n "$RUSTFLAGS" ]; then - RUSTFLAGS="$RUSTFLAGS --remap-path-prefix $crate_sources=/sources" + RUSTFLAGS="$RUSTFLAGS --remap-path-prefix $cratesio_sources=/sources" + RUSTFLAGS="$RUSTFLAGS --remap-path-prefix $git_sources=/sources" + log "RUSTFLAGS (updated): $RUSTFLAGS" - elif [ -n "$CARGO_BUILD_RUSTFLAGS" ]; then - CARGO_BUILD_RUSTFLAGS="$CARGO_BUILD_RUSTFLAGS --remap-path-prefix $crate_sources=/sources" - log "CARGO_BUILD_RUSTFLAGS (updated): $CARGO_BUILD_RUSTFLAGS" else - export CARGO_BUILD_RUSTFLAGS="--remap-path-prefix $crate_sources=/sources" + if [ -z "$CARGO_BUILD_RUSTFLAGS" ]; then + export CARGO_BUILD_RUSTFLAGS="" + fi + + CARGO_BUILD_RUSTFLAGS="$CARGO_BUILD_RUSTFLAGS --remap-path-prefix $cratesio_sources=/sources" + CARGO_BUILD_RUSTFLAGS="$CARGO_BUILD_RUSTFLAGS --remap-path-prefix $git_sources=/sources" + log "CARGO_BUILD_RUSTFLAGS (updated): $CARGO_BUILD_RUSTFLAGS" fi @@ -210,7 +217,7 @@ let mkdir -p target - # make sure that all source files are tagged as "recent" (since we write + # Make sure that all source files are tagged as "recent" (since we write # some stubs here and there) find . -type f -exec touch {} + @@ -365,7 +372,7 @@ let }; }; - # Unpacks all dependencies required to compile user's crate. + # Crates.io dependencies required to compile user's crate. # # As an output, for each dependency, this derivation produces a subdirectory # containing `.cargo-checksum.json` (required for Cargo to process the crate) @@ -380,18 +387,19 @@ let # something-else-1.2.3/src (-> /nix/store/...) # ... # ``` - # - # (note that the actual crate format is not document, but in practice it's a - # gzipped tar.) - unpackedDependencies = symlinkJoinPassViaFile { - name = "dependencies"; - - paths = - (map unpackCrateDependency crateDependencies) ++ - (map unpackGitDependency gitDependencies); + unpackedCratesIoDependencies = symlinkJoinPassViaFile { + name = "crates-io-dependencies"; + paths = (map unpackCratesIoDependency cratesIoDependencies); + }; + + # Git dependencies required to compile user's crate; follows same format as + # the crates.io dependencies above. + unpackedGitDependencies = symlinkJoinPassViaFile { + name = "git-dependencies"; + paths = (map unpackGitDependency gitDependencies); }; - unpackCrateDependency = { name, version, sha256 }: + unpackCratesIoDependency = { name, version, sha256 }: let crate = fetchurl { inherit sha256; diff --git a/config.nix b/config.nix index 43b5f52..321e45d 100644 --- a/config.nix +++ b/config.nix @@ -310,7 +310,7 @@ let # version and sha256 of the crate # Example: # [ { name = "wabt", version = "2.0.6", sha256 = "..." } ] - crateDependencies = libb.mkVersions buildPlanConfig.cargolock; + cratesIoDependencies = libb.mkVersions buildPlanConfig.cargolock; }; # config used when planning the builds