Merge pull request #11569 from MinaProtocol/ylecornec/check_opam_switch-compatible

check opam switch - compatible

Author: ylecornec

Makefile

@@ -69,7 +69,15 @@ ocaml_version:
 ocaml_word_size:
 	@if ! ocamlopt -config | grep "word_size:" | grep $(WORD_SIZE); then echo "invalid machine word size, expected $(WORD_SIZE)" ; exit 1; fi
 
-ocaml_checks: ocaml_version ocaml_word_size
+
+# Checks that the current opam switch contains the packages from opam.export at the same version.
+# This check is disabled in the pure nix environment (that does not use opam).
+check_opam_switch:
+ifneq ($(DISABLE_CHECK_OPAM_SWITCH), true)
+	check_opam_switch opam.export
+endif
+
+ocaml_checks: ocaml_version ocaml_word_size check_opam_switch
 
 libp2p_helper:
 	make -C src/app/libp2p_helper

README-dev.md

@@ -50,8 +50,8 @@ Refer to [/dev](/dev).
 
 Mina has a variety of opam and system dependencies.
 
-To get all the opam dependencies you need, you run `opam switch import
-src/opam.export`.
+To get all the opam dependencies you need, you run `opam switch import opam.export`.
+> *_NOTE:_*  The switch provides a `dune_wrapper` binary that you can use instead of dune, and will fail early if your switch becomes out of sync with the `opam.export` file.
 
 Some of our dependencies aren't taken from `opam`, and aren't integrated
 with `dune`, so you need to add them manually, by running `scripts/pin-external-packages.sh`.

buildkite/src/Constants/ContainerImages.dhall

@@ -2,10 +2,10 @@
 -- NOTE: minaToolchainStretch is also used for building Ubuntu Bionic packages in CI
 {
   toolchainBase = "codaprotocol/ci-toolchain-base:v3",
-  minaToolchainStretch = "gcr.io/o1labs-192920/mina-toolchain@sha256:d2b4e83f641b87b2aee4b981f374d524802a5535e5326ecea9ebe8ad2676faf3",
-  minaToolchainBuster = "gcr.io/o1labs-192920/mina-toolchain@sha256:8701d3a8ba2c3cf56aa07196a68ac51687466ebace570a4e725bd38157a36e97",
-  minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:f887e30966bc2747302f1f7e0e3baa8e08aad4b387469f0212a4c3534260019b",
-  minaToolchainFocal = "gcr.io/o1labs-192920/mina-toolchain@sha256:16ebfd20c2f22cb99dde2afd253daa5c91d254f34316943a54e09c1caedfbe55",
+  minaToolchainStretch = "gcr.io/o1labs-192920/mina-toolchain@sha256:d58f431177b453f3dddd48119d1ca86c9daf9c51f54f2daec5092bb5de674dc8",
+  minaToolchainBuster = "gcr.io/o1labs-192920/mina-toolchain@sha256:d2305beb4ed343113b79a6cbf88d25fbd4fb9b85913a2fe039b6a545cec63213",
+  minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:ab84f1d19d73ca67d31ee7e11eafd0b5a6bea3fb165b2f3f0469fb6af3efc9e3",
+  minaToolchainFocal = "gcr.io/o1labs-192920/mina-toolchain@sha256:68bd612b2615946740a4e06dd0e9c3a87fd074cea1456efc1a0bc6149409c923",
   delegationBackendToolchain = "gcr.io/o1labs-192920/delegation-backend-production@sha256:8ca5880845514ef56a36bf766a0f9de96e6200d61b51f80d9f684a0ec9c031f4",
   elixirToolchain = "elixir:1.10-alpine",
   nodeToolchain = "node:14.13.1-stretch-slim",

buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall

@@ -23,7 +23,7 @@ Pipeline.build
           S.strictlyStart (S.contains "dockerfiles/stages/2-"),
           S.strictlyStart (S.contains "dockerfiles/stages/3-"),
           S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact"),
-          S.strictly (S.contains "src/opam.export")
+          S.strictly (S.contains "opam.export")
         ],
         path = "Release",
         name = "MinaToolchainArtifact"

dockerfiles/stages/2-opam-deps

@@ -36,11 +36,11 @@ WORKDIR $HOME/$MINA_DIR
 ENV OPAMYES 1
 
 # --- Import Opam Switch
-# TODO: handle this opam work without cloning the full repository (directly pull src/opam.export)
-RUN opam switch import src/opam.export
+# TODO: handle this opam work without cloning the full repository (directly pull opam.export)
+RUN opam switch import opam.export
 
 # --- Pin external packages / submodules
-# TODO: Would be really nice to pull this script, the git submodules, and src/opam.export exclusively in this stage
+# TODO: Would be really nice to pull this script, the git submodules, and opam.export exclusively in this stage
 RUN eval $(opam config env) \
   && scripts/pin-external-packages.sh \
   && opam clean --logs -cs

nix/README.md

@@ -196,7 +196,7 @@ If you wish to update all the inputs of this flake, run `nix flake update` . If
 
 The "pure" build is performed with the help of
 [opam-nix](https://github.com/tweag/opam-nix). The switch is imported
-from `src/opam.export`, and then external deps (from `src/external`)
+from `opam.export`, and then external deps (from `src/external`)
 are added on top. Also, all in-tree Rust dependencies
 (`kimchi_bindings` in particular) are built as separate derivations
 using `rustPlatform`. Implicit native dependencies are taken from

nix/checks.nix

@@ -51,6 +51,7 @@ inputs: pkgs: {
     buildPhase = "make check-format";
     installPhase = "touch $out";
     meta.checkDescription = "that OCaml code is formatted properly";
+    DISABLE_CHECK_OPAM_SWITCH = "true";
   };
 
   # todo: libp2p_ipc

nix/impure-shell.nix

@@ -30,7 +30,7 @@ pkgs.mkShell {
       tput bold
       printf 'opam init --bare # Only if you are running opam for the first time\n'
       printf 'opam update\n'
-      printf 'opam switch import src/opam.export --strict\n'
+      printf 'opam switch import opam.export --strict\n'
       printf 'eval $(opam env)\n'
       printf './scripts/pin-external-packages.sh\n'
       tput sgr0

nix/ocaml.nix

@@ -15,7 +15,7 @@ let
     opam-nix.makeOpamRepoRec ../src/external; # Pin external packages
   repos = [ external-repo inputs.opam-repository ];
 
-  export = opam-nix.importOpam ../src/opam.export;
+  export = opam-nix.importOpam ../opam.export;
   external-packages =
     getAttrs [ "sodium" "capnp" "rpc_parallel" "async_kernel" "base58" ]
     (builtins.mapAttrs (_: last) (opam-nix.listRepo external-repo));
@@ -135,6 +135,7 @@ let
         GO_CAPNP_STD = "${pkgs.go-capnproto2.src}/std";
 
         MARLIN_PLONK_STUBS = "${pkgs.marlin_plonk_bindings_stubs}/lib";
+        DISABLE_CHECK_OPAM_SWITCH = "true";
         configurePhase = ''
           export MINA_ROOT="$PWD"
           patchShebangs .

src/opam.export renamed to opam.export

@@ -7,6 +7,7 @@ roots: [
   "bitstring.4.1.0"
   "camlp4.4.14+1"
   "capnp.3.4.0"
+  "check_opam_switch.~dev"
   "cohttp-async.5.0.0"
   "core_extended.v0.14.0"
   "extlib.1.7.8"
@@ -67,6 +68,7 @@ installed: [
   "caqti-async.1.3.0"
   "caqti-driver-postgresql.1.5.1"
   "charInfo_width.1.1.0"
+  "check_opam_switch.~dev"
   "cmdliner.1.0.3"
   "cohttp.5.0.0"
   "cohttp-async.5.0.0"
@@ -140,6 +142,7 @@ installed: [
   "merlin-extend.0.6.1"
   "mew.0.1.0"
   "mew_vi.0.5.0"
+  "minicli.5.0.2"
   "mmap.1.1.0"
   "num.1.1"
   "ocaml.4.14.0"
@@ -161,7 +164,10 @@ installed: [
   "octavius.1.2.2"
   "odoc.2.1.0"
   "odoc-parser.1.0.0"
+  "opam-core.2.1.2"
   "opam-depext.1.2.1"
+  "opam-file-format.2.1.4"
+  "opam-format.2.1.2"
   "parsexp.v0.14.2"
   "postgresql.5.0.0"
   "ppx_assert.v0.14.0"
@@ -241,8 +247,43 @@ installed: [
   "zed.3.1.0"
 ]
 pinned: [
+  "check_opam_switch.~dev"
   "graphql_ppx.1.2.2"
 ]
+package "check_opam_switch" {
+  opam-version: "2.0"
+  version: "~dev"
+  synopsis:
+    "A tool to check that the current opam environment is up to date with an opam.export file"
+  description:
+    "A tool to check that the current opam environment is up to date with an opam.export file"
+  maintainer: "[email protected]"
+  depends: [
+    "dune" {>= "3.3"}
+    "opam-core" {>= "2.0.0"}
+    "opam-format" {>= "2.0.0"}
+    "minicli" {>= "5.0.0"}
+    "odoc" {with-doc}
+  ]
+  build: [
+    ["dune" "subst"] {dev}
+    [
+      "dune"
+      "build"
+      "-p"
+      name
+      "-j"
+      jobs
+      "@install"
+      "@runtest" {with-test}
+      "@doc" {with-doc}
+    ]
+  ]
+  url {
+    src: "https://github.com/tweag/check_opam_switch/archive/d0aa49884e0f9fd4bbb2cd1a32b798a12f84b603.tar.gz"
+    checksum: "sha256=24ab29ea4aff9da9d649f0b577c5d4e27ce2bef51058e139965cc9be25494a46"
+  }
+}
 package "graphql_ppx" {
   opam-version: "2.0"
   version: "1.2.2"

scripts/select-opam-deps.sh

@@ -8,7 +8,7 @@ set -eou pipefail
 
 RES=()
 for lib in $@; do
-  RES+=($(cat src/opam.export | grep '"'"$lib"'\.' | awk -F'"' '{ print $2 }'))
+  RES+=($(cat opam.export | grep '"'"$lib"'\.' | awk -F'"' '{ print $2 }'))
 done
 
 echo "${RES[@]}"