Merge pull request #11570 from MinaProtocol/ylecornec/check_opam_switch-develop

check opam switch - develop

Author: ylecornec

Makefile

@@ -70,7 +70,15 @@ ocaml_version:
 ocaml_word_size:
 	@if ! ocamlopt -config | grep "word_size:" | grep $(WORD_SIZE); then echo "invalid machine word size, expected $(WORD_SIZE)" ; exit 1; fi
 
-ocaml_checks: ocaml_version ocaml_word_size
+
+# Checks that the current opam switch contains the packages from opam.export at the same version.
+# This check is disabled in the pure nix environment (that does not use opam).
+check_opam_switch:
+ifneq ($(DISABLE_CHECK_OPAM_SWITCH), true)
+	check_opam_switch opam.export
+endif
+
+ocaml_checks: ocaml_version ocaml_word_size check_opam_switch
 
 libp2p_helper:
 	make -C src/app/libp2p_helper

README-dev.md

@@ -50,8 +50,8 @@ Refer to [/dev](/dev).
 
 Mina has a variety of opam and system dependencies.
 
-To get all the opam dependencies you need, you run `opam switch import
-src/opam.export`.
+To get all the opam dependencies you need, you run `opam switch import opam.export`.
+> *_NOTE:_*  The switch provides a `dune_wrapper` binary that you can use instead of dune, and will fail early if your switch becomes out of sync with the `opam.export` file.
 
 Some of our dependencies aren't taken from `opam`, and aren't integrated
 with `dune`, so you need to add them manually, by running `scripts/pin-external-packages.sh`.

buildkite/src/Constants/ContainerImages.dhall

@@ -2,10 +2,10 @@
 -- NOTE: minaToolchainStretch is also used for building Ubuntu Bionic packages in CI
 {
   toolchainBase = "codaprotocol/ci-toolchain-base:v3",
-  minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:106a6c5a5a8a4df8b26464edf9b0bd1cec35124ecf7d8340af4561a54812c701",
-  minaToolchainBuster = "gcr.io/o1labs-192920/mina-toolchain@sha256:1fe11f2e4f6d7483411d0966509cb5d0b108d903e7dccee94b97a710be78ff40",
-  minaToolchainStretch = "gcr.io/o1labs-192920/mina-toolchain@sha256:56e8058904ada5081e0839468c7ac7c7d2438a49a452e9f263b271bc87946db5",
-  minaToolchainFocal = "gcr.io/o1labs-192920/mina-toolchain@sha256:421b0dcf6adbd2622264b45b1a5deadcc0ace69841fd484492cdd7e3bec330bd",
+  minaToolchainStretch = "gcr.io/o1labs-192920/mina-toolchain@sha256:36ed8f899635354fadf05a8504c054f4ca2a71950164c4aa8bea31f4f760afdb",
+  minaToolchainBuster = "gcr.io/o1labs-192920/mina-toolchain@sha256:6de0eb2aded56dd3f1c21df727e2d1ea53194095c0a45b025eaa93cc74166185",
+  minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:77444cd510dc40efd8c62e5c1a53485a05f3e42d4f883d2cfe74158847763af2",
+  minaToolchainFocal = "gcr.io/o1labs-192920/mina-toolchain@sha256:0b6167a26de97ca715bdcc4cbe759e3117d2fd0c0f5e03038a382fc25c60afcb",
   delegationBackendToolchain = "gcr.io/o1labs-192920/delegation-backend-production@sha256:8ca5880845514ef56a36bf766a0f9de96e6200d61b51f80d9f684a0ec9c031f4",
   elixirToolchain = "elixir:1.10-alpine",
   nodeToolchain = "node:14.13.1-stretch-slim",

buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall

@@ -23,7 +23,7 @@ Pipeline.build
           S.strictlyStart (S.contains "dockerfiles/stages/2-"),
           S.strictlyStart (S.contains "dockerfiles/stages/3-"),
           S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact"),
-          S.strictly (S.contains "src/opam.export")
+          S.strictly (S.contains "opam.export")
         ],
         path = "Release",
         name = "MinaToolchainArtifact"

dockerfiles/stages/2-opam-deps

@@ -36,11 +36,11 @@ WORKDIR $HOME/$MINA_DIR
 ENV OPAMYES 1
 
 # --- Import Opam Switch
-# TODO: handle this opam work without cloning the full repository (directly pull src/opam.export)
-RUN opam switch import src/opam.export
+# TODO: handle this opam work without cloning the full repository (directly pull opam.export)
+RUN opam switch import opam.export
 
 # --- Pin external packages / submodules
-# TODO: Would be really nice to pull this script, the git submodules, and src/opam.export exclusively in this stage
+# TODO: Would be really nice to pull this script, the git submodules, and opam.export exclusively in this stage
 RUN eval $(opam config env) \
   && scripts/pin-external-packages.sh \
   && opam clean --logs -cs

nix/README.md

@@ -196,7 +196,7 @@ If you wish to update all the inputs of this flake, run `nix flake update` . If
 
 The "pure" build is performed with the help of
 [opam-nix](https://github.com/tweag/opam-nix). The switch is imported
-from `src/opam.export`, and then external deps (from `src/external`)
+from `opam.export`, and then external deps (from `src/external`)
 are added on top. Also, all in-tree Rust dependencies
 (`kimchi_bindings` in particular) are built as separate derivations
 using `rustPlatform`. Implicit native dependencies are taken from

nix/checks.nix

@@ -51,6 +51,7 @@ inputs: pkgs: {
     buildPhase = "make check-format";
     installPhase = "touch $out";
     meta.checkDescription = "that OCaml code is formatted properly";
+    DISABLE_CHECK_OPAM_SWITCH = "true";
   };
 
   # todo: libp2p_ipc

nix/impure-shell.nix

@@ -32,7 +32,7 @@ pkgs.mkShell {
       tput bold
       printf 'opam init --bare # Only if you are running opam for the first time\n'
       printf 'opam update\n'
-      printf 'opam switch import src/opam.export --strict\n'
+      printf 'opam switch import opam.export --strict\n'
       printf 'eval $(opam env)\n'
       printf './scripts/pin-external-packages.sh\n'
       tput sgr0

nix/ocaml.nix

@@ -15,7 +15,7 @@ let
     opam-nix.makeOpamRepoRec ../src/external; # Pin external packages
   repos = [ external-repo inputs.opam-repository ];
 
-  export = opam-nix.importOpam ../src/opam.export;
+  export = opam-nix.importOpam ../opam.export;
   external-packages = pkgs.lib.getAttrs [ "sodium" "base58" ]
     (builtins.mapAttrs (_: pkgs.lib.last) (opam-nix.listRepo external-repo));
 
@@ -144,6 +144,7 @@ let
         # and copy it from within a dune rule
         # (see src/lib/crypto/kimchi_bindings/stubs/dune)
         MARLIN_PLONK_STUBS = "${pkgs.kimchi_bindings_stubs}";
+        DISABLE_CHECK_OPAM_SWITCH = "true";
 
         PLONK_WASM_NODEJS = "${pkgs.plonk_wasm}/nodejs";
         PLONK_WASM_WEB = "${pkgs.plonk_wasm}/web";

src/opam.export renamed to opam.export

@@ -7,6 +7,7 @@ roots: [
   "bitstring.4.1.0"
   "camlp4.4.14+1"
   "capnp.3.4.0"
+  "check_opam_switch.~dev"
   "cohttp-async.5.0.0"
   "core_extended.v0.14.0"
   "extlib.1.7.8"
@@ -68,6 +69,7 @@ installed: [
   "caqti-async.1.3.0"
   "caqti-driver-postgresql.1.5.1"
   "charInfo_width.1.1.0"
+  "check_opam_switch.~dev"
   "cmdliner.1.0.3"
   "cohttp.5.0.0"
   "cohttp-async.5.0.0"
@@ -142,6 +144,7 @@ installed: [
   "merlin-extend.0.6.1"
   "mew.0.1.0"
   "mew_vi.0.5.0"
+  "minicli.5.0.2"
   "mmap.1.1.0"
   "num.1.1"
   "ocaml.4.14.0"
@@ -163,7 +166,10 @@ installed: [
   "octavius.1.2.2"
   "odoc.2.1.0"
   "odoc-parser.1.0.0"
+  "opam-core.2.1.2"
   "opam-depext.1.2.1"
+  "opam-file-format.2.1.4"
+  "opam-format.2.1.2"
   "parsexp.v0.14.2"
   "postgresql.5.0.0"
   "ppx_assert.v0.14.0"
@@ -243,11 +249,46 @@ installed: [
   "zed.3.1.0"
 ]
 pinned: [
+  "check_opam_switch.~dev"
   "graphql_ppx.1.2.2"
   "capnp.3.4.0"
   "rpc_parallel.v0.14.0"
   "async_kernel.v0.14.0"
 ]
+package "check_opam_switch" {
+  opam-version: "2.0"
+  version: "~dev"
+  synopsis:
+    "A tool to check that the current opam environment is up to date with an opam.export file"
+  description:
+    "A tool to check that the current opam environment is up to date with an opam.export file"
+  maintainer: "[email protected]"
+  depends: [
+    "dune" {>= "3.3"}
+    "opam-core" {>= "2.0.0"}
+    "opam-format" {>= "2.0.0"}
+    "minicli" {>= "5.0.0"}
+    "odoc" {with-doc}
+  ]
+  build: [
+    ["dune" "subst"] {dev}
+    [
+      "dune"
+      "build"
+      "-p"
+      name
+      "-j"
+      jobs
+      "@install"
+      "@runtest" {with-test}
+      "@doc" {with-doc}
+    ]
+  ]
+  url {
+    src: "https://github.com/tweag/check_opam_switch/archive/d0aa49884e0f9fd4bbb2cd1a32b798a12f84b603.tar.gz"
+    checksum: "sha256=24ab29ea4aff9da9d649f0b577c5d4e27ce2bef51058e139965cc9be25494a46"
+  }
+}
 package "graphql_ppx" {
   opam-version: "2.0"
   version: "1.2.2"
@@ -278,6 +319,7 @@ package "graphql_ppx" {
          checksum: "sha256=8aff33988218c99a95a91fb04ef703a3b8cde0676b7a90ee72478959829867d8"
   }
 }
+
 package "capnp" {
   opam-version: "2.0"
   version: "3.4.0"

scripts/select-opam-deps.sh

@@ -8,7 +8,7 @@ set -eou pipefail
 
 RES=()
 for lib in $@; do
-  RES+=($(cat src/opam.export | grep '"'"$lib"'\.' | awk -F'"' '{ print $2 }'))
+  RES+=($(cat opam.export | grep '"'"$lib"'\.' | awk -F'"' '{ print $2 }'))
 done
 
 echo "${RES[@]}"