Skip to content

check opam switch - compatible #11569

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Aug 29, 2022
Merged

check opam switch - compatible #11569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
6b9a9ba
move opam.export to the root of the dune project
ylecornec Jul 26, 2022
28e9962
add check_opam_switch to opam.export
ylecornec Jul 26, 2022
e5150d0
check opam switch in makefile
ylecornec Jul 26, 2022
e71ce26
Update container images
ylecornec Jul 26, 2022
42ca239
update container images
ylecornec Jul 26, 2022
320d734
Disable check_opam_switch in nix checks
ylecornec Jul 27, 2022
35abc1b
Add comments related to check_opam_switch
ylecornec Jul 27, 2022
06e4956
Merge remote-tracking branch 'origin/compatible' into ylecornec/check…
ylecornec Aug 24, 2022
7adb65f
Update container images
ylecornec Aug 24, 2022
fcab071
Merge remote-tracking branch 'origin/compatible' into ylecornec/check…
ylecornec Aug 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,15 @@ ocaml_version:
ocaml_word_size:
@if ! ocamlopt -config | grep "word_size:" | grep $(WORD_SIZE); then echo "invalid machine word size, expected $(WORD_SIZE)" ; exit 1; fi

ocaml_checks: ocaml_version ocaml_word_size

# Checks that the current opam switch contains the packages from opam.export at the same version.
# This check is disabled in the pure nix environment (that does not use opam).
check_opam_switch:
ifneq ($(DISABLE_CHECK_OPAM_SWITCH), true)
check_opam_switch opam.export
endif

ocaml_checks: ocaml_version ocaml_word_size check_opam_switch

libp2p_helper:
make -C src/app/libp2p_helper
Expand Down
4 changes: 2 additions & 2 deletions README-dev.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@ Refer to [/dev](/dev).

Mina has a variety of opam and system dependencies.

To get all the opam dependencies you need, you run `opam switch import
src/opam.export`.
To get all the opam dependencies you need, you run `opam switch import opam.export`.
> *_NOTE:_* The switch provides a `dune_wrapper` binary that you can use instead of dune, and will fail early if your switch becomes out of sync with the `opam.export` file.

Some of our dependencies aren't taken from `opam`, and aren't integrated
with `dune`, so you need to add them manually, by running `scripts/pin-external-packages.sh`.
Expand Down
8 changes: 4 additions & 4 deletions buildkite/src/Constants/ContainerImages.dhall
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
-- NOTE: minaToolchainStretch is also used for building Ubuntu Bionic packages in CI
{
toolchainBase = "codaprotocol/ci-toolchain-base:v3",
minaToolchainStretch = "gcr.io/o1labs-192920/mina-toolchain@sha256:d2b4e83f641b87b2aee4b981f374d524802a5535e5326ecea9ebe8ad2676faf3",
minaToolchainBuster = "gcr.io/o1labs-192920/mina-toolchain@sha256:8701d3a8ba2c3cf56aa07196a68ac51687466ebace570a4e725bd38157a36e97",
minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:f887e30966bc2747302f1f7e0e3baa8e08aad4b387469f0212a4c3534260019b",
minaToolchainFocal = "gcr.io/o1labs-192920/mina-toolchain@sha256:16ebfd20c2f22cb99dde2afd253daa5c91d254f34316943a54e09c1caedfbe55",
minaToolchainStretch = "gcr.io/o1labs-192920/mina-toolchain@sha256:d58f431177b453f3dddd48119d1ca86c9daf9c51f54f2daec5092bb5de674dc8",
minaToolchainBuster = "gcr.io/o1labs-192920/mina-toolchain@sha256:d2305beb4ed343113b79a6cbf88d25fbd4fb9b85913a2fe039b6a545cec63213",
minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:ab84f1d19d73ca67d31ee7e11eafd0b5a6bea3fb165b2f3f0469fb6af3efc9e3",
minaToolchainFocal = "gcr.io/o1labs-192920/mina-toolchain@sha256:68bd612b2615946740a4e06dd0e9c3a87fd074cea1456efc1a0bc6149409c923",
delegationBackendToolchain = "gcr.io/o1labs-192920/delegation-backend-production@sha256:8ca5880845514ef56a36bf766a0f9de96e6200d61b51f80d9f684a0ec9c031f4",
elixirToolchain = "elixir:1.10-alpine",
nodeToolchain = "node:14.13.1-stretch-slim",
Expand Down
2 changes: 1 addition & 1 deletion buildkite/src/Jobs/Release/MinaToolchainArtifact.dhall
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ Pipeline.build
S.strictlyStart (S.contains "dockerfiles/stages/2-"),
S.strictlyStart (S.contains "dockerfiles/stages/3-"),
S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact"),
S.strictly (S.contains "src/opam.export")
S.strictly (S.contains "opam.export")
],
path = "Release",
name = "MinaToolchainArtifact"
Expand Down
6 changes: 3 additions & 3 deletions dockerfiles/stages/2-opam-deps
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ WORKDIR $HOME/$MINA_DIR
ENV OPAMYES 1

# --- Import Opam Switch
# TODO: handle this opam work without cloning the full repository (directly pull src/opam.export)
RUN opam switch import src/opam.export
# TODO: handle this opam work without cloning the full repository (directly pull opam.export)
RUN opam switch import opam.export

# --- Pin external packages / submodules
# TODO: Would be really nice to pull this script, the git submodules, and src/opam.export exclusively in this stage
# TODO: Would be really nice to pull this script, the git submodules, and opam.export exclusively in this stage
RUN eval $(opam config env) \
&& scripts/pin-external-packages.sh \
&& opam clean --logs -cs
2 changes: 1 addition & 1 deletion nix/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ If you wish to update all the inputs of this flake, run `nix flake update` . If

The "pure" build is performed with the help of
[opam-nix](https://github.com/tweag/opam-nix). The switch is imported
from `src/opam.export`, and then external deps (from `src/external`)
from `opam.export`, and then external deps (from `src/external`)
are added on top. Also, all in-tree Rust dependencies
(`kimchi_bindings` in particular) are built as separate derivations
using `rustPlatform`. Implicit native dependencies are taken from
Expand Down
1 change: 1 addition & 0 deletions nix/checks.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ inputs: pkgs: {
buildPhase = "make check-format";
installPhase = "touch $out";
meta.checkDescription = "that OCaml code is formatted properly";
DISABLE_CHECK_OPAM_SWITCH = "true";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not entirely convinced that we should skip the check here: using the wrong opam.export may give us different formatting, depending on the version of ocamlformat.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(btw I think the ocamlformat version should be enforced with an .ocamlformat file)

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For this check, the environment is managed via nix and not opam, so the check_opam_switch tool does not work in this case.

It should not be needed however as nix directly reads the opam.export file to setup the environment.

};

# todo: libp2p_ipc
Expand Down
2 changes: 1 addition & 1 deletion nix/impure-shell.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ pkgs.mkShell {
tput bold
printf 'opam init --bare # Only if you are running opam for the first time\n'
printf 'opam update\n'
printf 'opam switch import src/opam.export --strict\n'
printf 'opam switch import opam.export --strict\n'
printf 'eval $(opam env)\n'
printf './scripts/pin-external-packages.sh\n'
tput sgr0
Expand Down
3 changes: 2 additions & 1 deletion nix/ocaml.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ let
opam-nix.makeOpamRepoRec ../src/external; # Pin external packages
repos = [ external-repo inputs.opam-repository ];

export = opam-nix.importOpam ../src/opam.export;
export = opam-nix.importOpam ../opam.export;
external-packages =
getAttrs [ "sodium" "capnp" "rpc_parallel" "async_kernel" "base58" ]
(builtins.mapAttrs (_: last) (opam-nix.listRepo external-repo));
Expand Down Expand Up @@ -135,6 +135,7 @@ let
GO_CAPNP_STD = "${pkgs.go-capnproto2.src}/std";

MARLIN_PLONK_STUBS = "${pkgs.marlin_plonk_bindings_stubs}/lib";
DISABLE_CHECK_OPAM_SWITCH = "true";
configurePhase = ''
export MINA_ROOT="$PWD"
patchShebangs .
Expand Down
41 changes: 41 additions & 0 deletions src/opam.export → opam.export
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ roots: [
"bitstring.4.1.0"
"camlp4.4.14+1"
"capnp.3.4.0"
"check_opam_switch.~dev"
"cohttp-async.5.0.0"
"core_extended.v0.14.0"
"extlib.1.7.8"
Expand Down Expand Up @@ -67,6 +68,7 @@ installed: [
"caqti-async.1.3.0"
"caqti-driver-postgresql.1.5.1"
"charInfo_width.1.1.0"
"check_opam_switch.~dev"
"cmdliner.1.0.3"
"cohttp.5.0.0"
"cohttp-async.5.0.0"
Expand Down Expand Up @@ -140,6 +142,7 @@ installed: [
"merlin-extend.0.6.1"
"mew.0.1.0"
"mew_vi.0.5.0"
"minicli.5.0.2"
"mmap.1.1.0"
"num.1.1"
"ocaml.4.14.0"
Expand All @@ -161,7 +164,10 @@ installed: [
"octavius.1.2.2"
"odoc.2.1.0"
"odoc-parser.1.0.0"
"opam-core.2.1.2"
"opam-depext.1.2.1"
"opam-file-format.2.1.4"
"opam-format.2.1.2"
"parsexp.v0.14.2"
"postgresql.5.0.0"
"ppx_assert.v0.14.0"
Expand Down Expand Up @@ -241,8 +247,43 @@ installed: [
"zed.3.1.0"
]
pinned: [
"check_opam_switch.~dev"
"graphql_ppx.1.2.2"
]
package "check_opam_switch" {
opam-version: "2.0"
version: "~dev"
synopsis:
"A tool to check that the current opam environment is up to date with an opam.export file"
description:
"A tool to check that the current opam environment is up to date with an opam.export file"
maintainer: "[email protected]"
depends: [
"dune" {>= "3.3"}
"opam-core" {>= "2.0.0"}
"opam-format" {>= "2.0.0"}
"minicli" {>= "5.0.0"}
"odoc" {with-doc}
]
build: [
["dune" "subst"] {dev}
[
"dune"
"build"
"-p"
name
"-j"
jobs
"@install"
"@runtest" {with-test}
"@doc" {with-doc}
]
]
url {
src: "https://github.com/tweag/check_opam_switch/archive/d0aa49884e0f9fd4bbb2cd1a32b798a12f84b603.tar.gz"
checksum: "sha256=24ab29ea4aff9da9d649f0b577c5d4e27ce2bef51058e139965cc9be25494a46"
}
}
package "graphql_ppx" {
opam-version: "2.0"
version: "1.2.2"
Expand Down
2 changes: 1 addition & 1 deletion scripts/select-opam-deps.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ set -eou pipefail

RES=()
for lib in $@; do
RES+=($(cat src/opam.export | grep '"'"$lib"'\.' | awk -F'"' '{ print $2 }'))
RES+=($(cat opam.export | grep '"'"$lib"'\.' | awk -F'"' '{ print $2 }'))
done

echo "${RES[@]}"
Expand Down