diff --git a/.github/workflows/fast.yml b/.github/workflows/fast.yml index 6e340ae..3375af1 100644 --- a/.github/workflows/fast.yml +++ b/.github/workflows/fast.yml @@ -10,11 +10,11 @@ jobs: linux: strategy: matrix: - nixpkgs: [ nixpkgs, nixpkgs-21.05, nixpkgs-21.11 ] - runs-on: ubuntu-20.04 + nixpkgs: [ nixpkgs, nixpkgs-21.05, nixpkgs-22.05, nixpkgs-23.05 ] + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v16 + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v22 with: CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} - name: "Run tests" @@ -22,10 +22,10 @@ jobs: env: nixpkgs: ${{ matrix.nixpkgs }} darwin: - runs-on: macos-10.15 + runs-on: macos-latest steps: - - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v16 + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v22 with: CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} - name: "Run tests" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a2d7acd..f345b16 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,11 +8,11 @@ jobs: linux: strategy: matrix: - nixpkgs: [ nixpkgs, nixpkgs-21.05, nixpkgs-21.11 ] - runs-on: ubuntu-20.04 + nixpkgs: [ nixpkgs, nixpkgs-21.05, nixpkgs-22.05, nixpkgs-23.05 ] + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v16 + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v22 with: CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} - name: "Run tests" @@ -20,10 +20,10 @@ jobs: env: nixpkgs: ${{ matrix.nixpkgs }} darwin: - runs-on: macos-10.15 + runs-on: macos-latest steps: - - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v16 + - uses: actions/checkout@v3 + - uses: cachix/install-nix-action@v22 with: CACHIX_SIGNING_KEY: ${{ secrets.CACHIX_SIGNING_KEY }} - name: "Run tests" diff --git a/build.nix b/build.nix index 8ab8d97..aeff784 100644 --- a/build.nix +++ b/build.nix @@ -62,7 +62,8 @@ , writeText , runCommandLocal , remarshal -, crateDependencies +, formats +, cratesIoDependencies , zstd , fetchurl , lndir @@ -72,7 +73,7 @@ let builtinz = builtins // import ./builtins - { inherit lib writeText remarshal runCommandLocal; }; + { inherit lib writeText remarshal runCommandLocal formats; }; drvAttrs = { name = "${pname}-${version}"; @@ -83,15 +84,18 @@ let postInstall ; - crate_sources = unpackedDependencies; + cratesio_sources = unpackedCratesIoDependencies; + git_sources = unpackedGitDependencies; # The cargo config with source replacement. Replaces both crates.io crates # and git dependencies. - cargoconfig = builtinz.toTOML { + cargoconfig = builtinz.writeTOML "config" { source = { - crates-io = { replace-with = "nix-sources"; }; - nix-sources = { - directory = unpackedDependencies; + crates-io = { + directory = unpackedCratesIoDependencies; + }; + git = { + directory = unpackedGitDependencies; }; } // lib.listToAttrs ( map @@ -107,7 +111,7 @@ let name = "${e.url}${key}"; value = lib.filterAttrs (n: _: n == "rev" || n == "tag" || n == "branch") e // { git = e.url; - replace-with = "nix-sources"; + replace-with = "git"; }; } ) @@ -187,7 +191,6 @@ let log "RUST_TEST_THREADS: $RUST_TEST_THREADS" log "cargo_bins_jq_filter: $cargo_bins_jq_filter" log "cargo_build_output_json (created): $cargo_build_output_json" - log "crate_sources: $crate_sources" log "RUSTFLAGS: $RUSTFLAGS" log "CARGO_BUILD_RUSTFLAGS: $CARGO_BUILD_RUSTFLAGS" @@ -195,13 +198,18 @@ let # Remove the source path(s) in Rust if [ -n "$RUSTFLAGS" ]; then - RUSTFLAGS="$RUSTFLAGS --remap-path-prefix $crate_sources=/sources" + RUSTFLAGS="$RUSTFLAGS --remap-path-prefix $cratesio_sources=/sources" + RUSTFLAGS="$RUSTFLAGS --remap-path-prefix $git_sources=/sources" + log "RUSTFLAGS (updated): $RUSTFLAGS" - elif [ -n "$CARGO_BUILD_RUSTFLAGS" ]; then - CARGO_BUILD_RUSTFLAGS="$CARGO_BUILD_RUSTFLAGS --remap-path-prefix $crate_sources=/sources" - log "CARGO_BUILD_RUSTFLAGS (updated): $CARGO_BUILD_RUSTFLAGS" else - export CARGO_BUILD_RUSTFLAGS="--remap-path-prefix $crate_sources=/sources" + if [ -z "$CARGO_BUILD_RUSTFLAGS" ]; then + export CARGO_BUILD_RUSTFLAGS="" + fi + + CARGO_BUILD_RUSTFLAGS="$CARGO_BUILD_RUSTFLAGS --remap-path-prefix $cratesio_sources=/sources" + CARGO_BUILD_RUSTFLAGS="$CARGO_BUILD_RUSTFLAGS --remap-path-prefix $git_sources=/sources" + log "CARGO_BUILD_RUSTFLAGS (updated): $CARGO_BUILD_RUSTFLAGS" fi @@ -209,7 +217,7 @@ let mkdir -p target - # make sure that all source files are tagged as "recent" (since we write + # Make sure that all source files are tagged as "recent" (since we write # some stubs here and there) find . -type f -exec touch {} + @@ -235,7 +243,7 @@ let export CARGO_HOME=''${CARGO_HOME:-$PWD/.cargo-home} mkdir -p $CARGO_HOME - echo "$cargoconfig" > $CARGO_HOME/config + cp "$cargoconfig" $CARGO_HOME/config runHook postConfigure ''; @@ -364,7 +372,7 @@ let }; }; - # Unpacks all dependencies required to compile user's crate. + # Crates.io dependencies required to compile user's crate. # # As an output, for each dependency, this derivation produces a subdirectory # containing `.cargo-checksum.json` (required for Cargo to process the crate) @@ -379,18 +387,19 @@ let # something-else-1.2.3/src (-> /nix/store/...) # ... # ``` - # - # (note that the actual crate format is not document, but in practice it's a - # gzipped tar.) - unpackedDependencies = symlinkJoinPassViaFile { - name = "dependencies"; - - paths = - (map unpackCrateDependency crateDependencies) ++ - (map unpackGitDependency gitDependencies); + unpackedCratesIoDependencies = symlinkJoinPassViaFile { + name = "crates-io-dependencies"; + paths = (map unpackCratesIoDependency cratesIoDependencies); + }; + + # Git dependencies required to compile user's crate; follows same format as + # the crates.io dependencies above. + unpackedGitDependencies = symlinkJoinPassViaFile { + name = "git-dependencies"; + paths = (map unpackGitDependency gitDependencies); }; - unpackCrateDependency = { name, version, sha256 }: + unpackCratesIoDependency = { name, version, sha256 }: let crate = fetchurl { inherit sha256; diff --git a/builtins/default.nix b/builtins/default.nix index 755647a..7bcf40e 100644 --- a/builtins/default.nix +++ b/builtins/default.nix @@ -3,12 +3,52 @@ , writeText , runCommandLocal , remarshal +, formats }: rec { - toTOML = import ./to-toml.nix { inherit lib; }; - writeTOML = name: attrs: writeText name (toTOML attrs); + # Serializes given attrset into a TOML file. + # + # Usage: + # writeTOML path attrset + # + # On newer nixpkgs, this function invokes `lib.formats.toml` that nowadays + # handles all TOML documents properly. + # + # On older nixpkgs, where that serializer doesn't work correctly¹, we rely on + # a custom implementation (with its own tiny shortcomings²). + # + # TODO remove our custom serializer after nixpkgs v23 becomes more widely + # adopted + # + # ¹ e.g. cases like `[targets."cfg(\"something\")"]` are translated badly + # ² https://github.com/nix-community/naersk/issues/263 + writeTOML = + let + our-impl = + let + to-toml = import ./to-toml.nix { + inherit lib; + }; + + in + name: value: + runCommandLocal name { + value = to-toml value; + passAsFile = [ "value" ]; + } '' + cp "$valuePath" "$out" + cat "$out" + ''; + + nixpkgs-impl = (formats.toml { }).generate; + + in + if builtins.compareVersions lib.version "22.11" <= 0 then + our-impl + else + nixpkgs-impl; readTOML = usePure: f: if usePure then diff --git a/config.nix b/config.nix index 43b5f52..321e45d 100644 --- a/config.nix +++ b/config.nix @@ -310,7 +310,7 @@ let # version and sha256 of the crate # Example: # [ { name = "wabt", version = "2.0.6", sha256 = "..." } ] - crateDependencies = libb.mkVersions buildPlanConfig.cargolock; + cratesIoDependencies = libb.mkVersions buildPlanConfig.cargolock; }; # config used when planning the builds diff --git a/default.nix b/default.nix index 35de254..61b44ec 100644 --- a/default.nix +++ b/default.nix @@ -5,6 +5,7 @@ , lib , lndir , remarshal +, formats , rsync , runCommandLocal , rustc @@ -15,10 +16,10 @@ }@defaultBuildAttrs: let - libb = import ./lib.nix { inherit lib writeText runCommandLocal remarshal; }; + libb = import ./lib.nix { inherit lib writeText runCommandLocal remarshal formats; }; builtinz = builtins // import ./builtins - { inherit lib writeText remarshal runCommandLocal; }; + { inherit lib writeText remarshal runCommandLocal formats; }; mkConfig = arg: import ./config.nix { inherit lib arg libb builtinz; }; diff --git a/lib.nix b/lib.nix index a615bf2..33a5875 100644 --- a/lib.nix +++ b/lib.nix @@ -1,8 +1,8 @@ -{ lib, writeText, runCommandLocal, remarshal }: +{ lib, writeText, runCommandLocal, remarshal, formats }: let builtinz = builtins // import ./builtins - { inherit lib writeText remarshal runCommandLocal; }; + { inherit lib writeText remarshal runCommandLocal formats; }; in rec { @@ -121,14 +121,13 @@ rec # A very minimal 'src' which makes cargo happy nonetheless dummySrc = - { cargoconfig # string + { cargoconfig # path , cargotomls # list , cargolock # attrset , copySources # list of paths that should be copied to the output , copySourcesFrom # path from which to copy ${copySources} }: let - config = writeText "config" cargoconfig; cargolock' = builtinz.writeTOML "Cargo.lock" cargolock; fixupCargoToml = cargotoml: @@ -154,7 +153,7 @@ rec { inherit copySources copySourcesFrom cargotomlss; } '' mkdir -p $out/.cargo - ${lib.optionalString (! isNull cargoconfig) "cp ${config} $out/.cargo/config"} + ${lib.optionalString (! isNull cargoconfig) "cp ${cargoconfig} $out/.cargo/config"} cp ${cargolock'} $out/Cargo.lock for tuple in $cargotomlss; do diff --git a/nix/sources.json b/nix/sources.json index ecf2b7d..d2a14ef 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -1,14 +1,14 @@ { "agent-rs": { - "branch": "paulliu/add-cargo-lock", + "branch": "main", "description": "A collection of libraries and tools for building software around the Internet Computer, in Rust.", - "homepage": "https://sdk.dfinity.org/", - "owner": "ninegua", + "homepage": "", + "owner": "dfinity", "repo": "agent-rs", - "rev": "4a22e590516bc79ec3c75a320f7941e7762ea098", - "sha256": "0sacddc34nlfgldqghlwchgzjki177h5dsgpmdv70cm8hfy0sg7l", + "rev": "6d923cbc918852ef5bfaeb08e63c86580aa80ffe", + "sha256": "12y28nmv9af8sj55s998wydy08h32gvxr9dbhfxhfilxdairh0a2", "type": "tarball", - "url": "https://github.com/ninegua/agent-rs/archive/4a22e590516bc79ec3c75a320f7941e7762ea098.tar.gz", + "url": "https://github.com/dfinity/agent-rs/archive/6d923cbc918852ef5bfaeb08e63c86580aa80ffe.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "fenix": { @@ -65,10 +65,10 @@ "homepage": null, "owner": "NixOS", "repo": "nixpkgs", - "rev": "6c2318e451fc0eaf338fb461d9bfcc99869de758", - "sha256": "1djwkvjnn26v0xw1swwh7dgmi0yl0b1kb8kansrdwk0jhhqbl7zq", + "rev": "67bcf01c471217a9a1ac7e8aac3e5dde182ed6e9", + "sha256": "1wpmq4svsvb1scn5fmm5m0pq4wrdq4chn569x2v7qdfbqjir4sd0", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/6c2318e451fc0eaf338fb461d9bfcc99869de758.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/67bcf01c471217a9a1ac7e8aac3e5dde182ed6e9.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs-21.05": { @@ -83,16 +83,28 @@ "url": "https://github.com/NixOS/nixpkgs/archive/6d684ea3adef590a2174f2723134e1ea377272d2.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, - "nixpkgs-21.11": { - "branch": "release-21.11", + "nixpkgs-22.05": { + "branch": "release-22.05", "description": "Nix Packages collection", "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "432864f33c84af53192d4b23ee5871697e57f094", - "sha256": "0sqmasma9qin7nhfzv50d1la1mi1yazn7gx2lrxi9j61i9j8l3zm", + "rev": "380be19fbd2d9079f677978361792cb25e8a3635", + "sha256": "154x9swf494mqwi4z8nbq2f0sp8pwp4fvx51lqzindjfbb9yxxv5", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/432864f33c84af53192d4b23ee5871697e57f094.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/380be19fbd2d9079f677978361792cb25e8a3635.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs-23.05": { + "branch": "release-23.05", + "description": "Nix Packages collection", + "homepage": "", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b81af66deb21f73a70c67e5ea189568af53b1e8c", + "sha256": "172i5kzn8qja3k0rz9wfj86fgbw2ivc68yw3ra8g86byqc1rcw1m", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/b81af66deb21f73a70c67e5ea189568af53b1e8c.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs-mozilla": { @@ -119,6 +131,18 @@ "url": "https://github.com/nushell/nushell/archive/85bfdca578157072e51e6972d370cfe63b0fda77.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, + "probe-rs": { + "branch": "master", + "description": "A debugging toolset and library for debugging embedded ARM and RISC-V targets on a separate host", + "homepage": "https://probe.rs", + "owner": "probe-rs", + "repo": "probe-rs", + "rev": "51fa324aef9f7c413988a3d18052b1bbc278a4c5", + "sha256": "0zb9s80hsz83ngjngs9cllp7gf8xq9jz0m3lwdhf08x3cp3bj6fd", + "type": "tarball", + "url": "https://github.com/probe-rs/probe-rs/archive/51fa324aef9f7c413988a3d18052b1bbc278a4c5.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "ripgrep-all": { "branch": "master", "description": "rga: ripgrep, but also search in PDFs, E-Books, Office documents, zip, tar.gz, etc.", diff --git a/test/default.nix b/test/default.nix index 75097f6..ede5b1c 100644 --- a/test/default.nix +++ b/test/default.nix @@ -2,9 +2,25 @@ let sources = import ../nix/sources.nix; - pkgs = import ../nix { - inherit system nixpkgs; - }; + pkgs = + let + pkgs' = import ../nix { + inherit system nixpkgs; + }; + + older-pkgs = import ../nix { + inherit system; + + nixpkgs = "nixpkgs-21.05"; + }; + + in + pkgs' // { + # Some of our tests use dynamically-built Git repositories that fail extra + # security checks introduced in newer Git versions - so for the time being + # let's pin our test-Git to an older version. + git = older-pkgs.git; + }; naersk = pkgs.callPackage ../default.nix { inherit (pkgs.rustPackages) cargo rustc; diff --git a/test/slow/agent-rs/default.nix b/test/slow/agent-rs/default.nix index 1b8452c..5a56ee7 100644 --- a/test/slow/agent-rs/default.nix +++ b/test/slow/agent-rs/default.nix @@ -1,5 +1,18 @@ -{ sources, naersk, pkgs, ... }: +{ sources, pkgs, ... }: +let + fenix = import sources.fenix { }; + toolchain = fenix.fromToolchainFile { + file = "${sources.agent-rs}/rust-toolchain.toml"; + sha256 = "sha256-DzNEaW724O8/B8844tt5AVHmSjSQ3cmzlU4BP90oRlY="; + }; + + naersk = pkgs.callPackage ../../../default.nix { + cargo = toolchain; + rustc = toolchain; + }; + +in naersk.buildPackage { src = sources.agent-rs; diff --git a/test/slow/default.nix b/test/slow/default.nix index d58dff1..81151d3 100644 --- a/test/slow/default.nix +++ b/test/slow/default.nix @@ -2,6 +2,7 @@ args: { agent-rs = import ./agent-rs args; lorri = import ./lorri args; nushell = import ./nushell args; + probe-rs = import ./probe-rs args; ripgrep-all = import ./ripgrep-all args; rustlings = import ./rustlings args; talent-plan = import ./talent-plan args; diff --git a/test/slow/probe-rs/default.nix b/test/slow/probe-rs/default.nix new file mode 100644 index 0000000..62b4677 --- /dev/null +++ b/test/slow/probe-rs/default.nix @@ -0,0 +1,40 @@ +{ sources, pkgs, ... }: +let + fenix = import sources.fenix { }; + + toolchain = (fenix.toolchainOf { + channel = "nightly"; + date = "2023-07-01"; + sha256 = "sha256-pWd4tAHP4QWGC3CKWZzDjzYANxATC6CGRmKuP2ZZv5k="; + }).toolchain; + + naersk = pkgs.callPackage ../../../default.nix { + cargo = toolchain; + rustc = toolchain; + }; + + app = naersk.buildPackage { + src = sources.probe-rs; + + buildInputs = with pkgs; [ + pkg-config + libusb1 + openssl + ] ++ lib.optionals stdenv.isDarwin [ + darwin.DarwinTools + darwin.apple_sdk.frameworks.AppKit + ]; + }; + +in +if builtins.compareVersions pkgs.lib.version "22.11" <= 0 then + # Executing this test requires nixpkgs > 22.11 due to changes to the TOML + # serialization function. + # + # See `writeTOML` in this repository for more details. + true +else + pkgs.runCommand "probe-rs-test" + { + buildInputs = [ app ]; + } "rtthost --help && touch $out"