From 24af67f629119f95c730edae832bcd260ad18fd0 Mon Sep 17 00:00:00 2001 From: James Hinshelwood Date: Fri, 6 Oct 2023 18:07:07 +0100 Subject: [PATCH 01/17] Make Scilla compatible with Zilliqa 2 * We add another binary which exposes the `scilla-server` JSON-RPC interface over HTTP. * The state IPC calls now make requests over HTTP, rather than via a unix domain socket. Note that this is a breaking change, meaning this version of Scilla will NOT work with Zilliqa 1. * All bytes in IPC calls are encoded into base64. Previously, we would include invalid UTF-8 strings in JSON-RPC requests, which resulted in invalid JSON. --- docker/Dockerfile | 3 +- package.json | 4 +- scilla.opam | 2 + src/eval/Disambiguator.ml | 4 +- src/eval/StateIPCClient.ml | 58 ++++++++++++++----------- src/eval/StateIPCIdl.ml | 6 +-- src/eval/dune | 2 +- src/runners/dune | 8 ++-- src/runners/scilla_server_http.ml | 72 +++++++++++++++++++++++++++++++ 9 files changed, 121 insertions(+), 38 deletions(-) create mode 100644 src/runners/scilla_server_http.ml diff --git a/docker/Dockerfile b/docker/Dockerfile index 42c1671b1..c1a4a88ed 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,6 +18,7 @@ RUN apt-get update -y \ git \ lcov \ libcurl4-openssl-dev \ + libev-dev \ libgmp-dev \ libpcre3-dev \ libssl-dev \ @@ -119,9 +120,9 @@ FROM ubuntu:22.04 RUN apt-get update -y \ && apt-get install -y build-essential \ + libev-dev \ libgmp-dev ARG SOURCE_DIR="/scilla/${MAJOR_VERSION}" COPY --from=builder ${SOURCE_DIR} ${SOURCE_DIR} - diff --git a/package.json b/package.json index 2c36fcfdf..841862b85 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,9 @@ "@opam/ppx_deriving_rpc": ">=6.0.0 <10.0.0", "@opam/secp256k1": ">=0.4.4 <0.5.0", "@opam/stdint": ">=0.5.1 <0.8.0", - "@opam/yojson": ">=1.7.0 <2.1.0" + "@opam/yojson": ">=1.7.0 <2.1.0", + "@opam/opium": ">=0.20.0 <1.0.0", + "@opam/ezcurl": ">=0.2.4 <0.3.0" }, "devDependencies": { "@opam/merlin": "*", diff --git a/scilla.opam b/scilla.opam index 601c115dd..6b8d4d718 100644 --- a/scilla.opam +++ b/scilla.opam @@ -49,6 +49,8 @@ depends: [ "seq" "stdint" {>= "0.5.1" & < "0.8~"} "yojson" {>= "1.7.0" & < "2.1~"} + "opium" {>= "0.20.0" & < "1.0.0"} + "ezcurl" {>= "0.2.4" & < "0.3.0"} ] build: [ [ "./scripts/build_deps.sh" ] diff --git a/src/eval/Disambiguator.ml b/src/eval/Disambiguator.ml index f14242069..83033a748 100644 --- a/src/eval/Disambiguator.ml +++ b/src/eval/Disambiguator.ml @@ -808,7 +808,7 @@ module InputStateService = struct try let encoder = Pbrt.Encoder.create () in Ipcmessage_pb.encode_proto_scilla_query query encoder; - Bytes.to_string @@ Pbrt.Encoder.to_bytes encoder + Base64.encode_exn @@ Bytes.to_string @@ Pbrt.Encoder.to_bytes encoder with e -> fatal_error (mk_error0 ~kind:(Exn.to_string e) ?inst:None) let decode_serialized_value value = @@ -838,7 +838,7 @@ module InputStateService = struct in match res with | Some (true, res') -> - let decoded_pb = decode_serialized_value (Bytes.of_string res') in + let decoded_pb = decode_serialized_value (Bytes.of_string (Base64.decode_exn res')) in let res'' = deserialize_value decoded_pb tp this_address in Some res'' | Some (false, _) | None -> None diff --git a/src/eval/StateIPCClient.ml b/src/eval/StateIPCClient.ml index 25802bbb5..7b913f71b 100644 --- a/src/eval/StateIPCClient.ml +++ b/src/eval/StateIPCClient.ml @@ -51,26 +51,32 @@ let ipcclient_exn_wrapper thunk = try thunk () with | Core_unix.Unix_error (_, s1, s2) -> fail0 ~kind:("StateIPCClient: Unix error: " ^ s1 ^ s2) ?inst:None - | _ -> + | e -> + let e = Exn.to_string e in + print_endline (Printf.sprintf "error making JSON-RPC call: %s" e); fail0 ~kind:"StateIPCClient: Unexpected error making JSON-RPC call" ?inst:None -let binary_rpc ~socket_addr (call : Rpc.call) : Rpc.response M.t = - let socket = - Core_unix.socket ~domain:Core_unix.PF_UNIX ~kind:Core_unix.SOCK_STREAM - ~protocol:0 () - in - Core_unix.connect socket ~addr:(Core_unix.ADDR_UNIX socket_addr); - let ic = Core_unix.in_channel_of_descr socket in - let oc = Core_unix.out_channel_of_descr socket in +let http_rpc ~socket_addr (call : Rpc.call) : Rpc.response M.t = let msg_buf = Jsonrpc.string_of_call ~version:Jsonrpc.V2 call in - DebugMessage.plog (Printf.sprintf "Sending: %s\n" msg_buf); - (* Send data to the socket. *) - let _ = send_delimited oc msg_buf in - (* Get response. *) - let response = Caml.input_line ic in - Core_unix.close socket; - DebugMessage.plog (Printf.sprintf "Response: %s\n" response); + print_endline (Printf.sprintf "Sending: %s\n" msg_buf); + let exception Http_error of string in + let response = + match Ezcurl.post ~headers:["content-type", "application/json"] ~content:(`String msg_buf) ~params:[] ~url:socket_addr () with + | Ok response -> response + | Error (_, err) -> ( + print_endline (Printf.sprintf "error calling RPC: %s" err); + raise (Http_error (Printf.sprintf "error calling RPC: %s" err)) + ) + in + + let response = if response.code = 200 then response.body else ( + print_endline (Printf.sprintf "error response from RPC: code: %d, body: %s" response.code response.body); + raise (Http_error "error response from RPC") + ) + in + + print_endline (Printf.sprintf "Response: %s\n" response); M.return @@ Jsonrpc.response_of_string response (* Encode a literal into bytes, opaque to the backend storage. *) @@ -137,7 +143,7 @@ let encode_serialized_value value = try let encoder = Pbrt.Encoder.create () in Ipcmessage_pb.encode_proto_scilla_val value encoder; - pure @@ Bytes.to_string @@ Pbrt.Encoder.to_bytes encoder + pure @@ Base64.encode_exn @@ Bytes.to_string @@ Pbrt.Encoder.to_bytes encoder with e -> fail0 ~kind:(Exn.to_string e) ?inst:None let decode_serialized_value value = @@ -150,7 +156,7 @@ let encode_serialized_query query = try let encoder = Pbrt.Encoder.create () in Ipcmessage_pb.encode_proto_scilla_query query encoder; - pure @@ Bytes.to_string @@ Pbrt.Encoder.to_bytes encoder + pure @@ Base64.encode_exn @@ Bytes.to_string @@ Pbrt.Encoder.to_bytes encoder with e -> fail0 ~kind:(Exn.to_string e) ?inst:None (* Fetch from a field. "keys" is empty when fetching non-map fields or an entire Map field. @@ -168,14 +174,14 @@ let fetch ~socket_addr ~fname ~keys ~tp = let%bind q' = encode_serialized_query q in let%bind res = let thunk () = - translate_res @@ IPCClient.fetch_state_value (binary_rpc ~socket_addr) q' + translate_res @@ IPCClient.fetch_state_value (http_rpc ~socket_addr) q' in ipcclient_exn_wrapper thunk in match res with | true, res' -> let%bind tp' = TypeUtilities.map_access_type tp (List.length keys) in - let%bind decoded_pb = decode_serialized_value (Bytes.of_string res') in + let%bind decoded_pb = decode_serialized_value (Bytes.of_string (Base64.decode_exn res')) in let%bind res'' = deserialize_value decoded_pb tp' in pure @@ Some res'' | false, _ -> pure None @@ -211,7 +217,7 @@ let external_fetch ~socket_addr ~caddr ~fname ~keys ~ignoreval = let%bind res = let thunk () = translate_res - @@ IPCClient.fetch_ext_state_value (binary_rpc ~socket_addr) caddr q' + @@ IPCClient.fetch_ext_state_value (http_rpc ~socket_addr) caddr q' in ipcclient_exn_wrapper thunk in @@ -226,7 +232,7 @@ let external_fetch ~socket_addr ~caddr ~fname ~keys ~ignoreval = let%bind tp' = TypeUtilities.map_access_type stored_typ (List.length keys) in - let%bind decoded_pb = decode_serialized_value (Bytes.of_string res') in + let%bind decoded_pb = decode_serialized_value (Bytes.of_string (Base64.decode_exn res')) in let%bind res'' = deserialize_value decoded_pb tp' in pure @@ (Some res'', Some stored_typ) | false, _, _ -> pure (None, None) @@ -247,7 +253,7 @@ let update ~socket_addr ~fname ~keys ~value ~tp = let%bind () = let thunk () = translate_res - @@ IPCClient.update_state_value (binary_rpc ~socket_addr) q' value' + @@ IPCClient.update_state_value (http_rpc ~socket_addr) q' value' in ipcclient_exn_wrapper thunk in @@ -267,7 +273,7 @@ let is_member ~socket_addr ~fname ~keys ~tp = let%bind q' = encode_serialized_query q in let%bind res = let thunk () = - translate_res @@ IPCClient.fetch_state_value (binary_rpc ~socket_addr) q' + translate_res @@ IPCClient.fetch_state_value (http_rpc ~socket_addr) q' in ipcclient_exn_wrapper thunk in @@ -290,7 +296,7 @@ let remove ~socket_addr ~fname ~keys ~tp = let%bind () = let thunk () = translate_res - @@ IPCClient.update_state_value (binary_rpc ~socket_addr) q' dummy_val + @@ IPCClient.update_state_value (http_rpc ~socket_addr) q' dummy_val in ipcclient_exn_wrapper thunk in @@ -304,7 +310,7 @@ let fetch_bcinfo ~socket_addr ~query_name ~query_args = let%bind res = let thunk () = translate_res - @@ IPCClient.fetch_bcinfo (binary_rpc ~socket_addr) query_name query_args + @@ IPCClient.fetch_bcinfo (http_rpc ~socket_addr) query_name query_args in ipcclient_exn_wrapper thunk in diff --git a/src/eval/StateIPCIdl.ml b/src/eval/StateIPCIdl.ml index b179c0423..29165c965 100644 --- a/src/eval/StateIPCIdl.ml +++ b/src/eval/StateIPCIdl.ml @@ -69,12 +69,12 @@ module IPCIdl (R : RPC) = struct let return_update = Param.mk Rpc.Types.unit let fetch_state_value = - declare "fetchStateValue" + declare "fetchStateValueB64" [ "Fetch state value from blockchain" ] (query @-> returning return_fetch RPCError.err) let fetch_ext_state_value = - declare "fetchExternalStateValue" + declare "fetchExternalStateValueB64" [ "Fetch state value of another contract from the blockchain" ] (addr @-> query @-> returning return_ext_fetch RPCError.err) @@ -99,7 +99,7 @@ module IPCIdl (R : RPC) = struct @-> returning return_update RPCError.err) let update_state_value = - declare "updateStateValue" + declare "updateStateValueB64" [ "Update state value in blockchain" ] (query @-> value @-> returning return_update RPCError.err) end diff --git a/src/eval/dune b/src/eval/dune index 19a291c24..10e174075 100644 --- a/src/eval/dune +++ b/src/eval/dune @@ -4,7 +4,7 @@ (wrapped true) (modes byte native) (libraries core core_unix.sys_unix angstrom stdint yojson cryptokit - scilla_base rpclib unix rpclib.json rresult ocaml-protoc) + scilla_base rpclib unix rpclib.json rresult ocaml-protoc ezcurl) (preprocess (pps ppx_sexp_conv ppx_let bisect_ppx --conditional ppx_deriving_rpc ppx_deriving.show ppx_compare)) diff --git a/src/runners/dune b/src/runners/dune index c5d7ad6cd..e78418762 100644 --- a/src/runners/dune +++ b/src/runners/dune @@ -1,14 +1,14 @@ (executables (names scilla_runner eval_runner type_checker scilla_checker scilla_server - disambiguate_state_json scilla_fmt scilla_merger) + disambiguate_state_json scilla_fmt scilla_merger scilla_server_http) (public_names scilla-runner eval-runner type-checker scilla-checker - scilla-server disambiguate_state_json scilla-fmt scilla-merger) + scilla-server disambiguate_state_json scilla-fmt scilla-merger scilla-server-http) (package scilla) (modules scilla_runner eval_runner type_checker scilla_checker scilla_server - disambiguate_state_json scilla_fmt scilla_merger) + disambiguate_state_json scilla_fmt scilla_merger scilla_server_http) (libraries core core_unix.command_unix angstrom yojson cryptokit fileutils scilla_base scilla_eval scilla_server_lib scilla_crypto scilla_format - scilla_merge cmdliner) + scilla_merge cmdliner opium) (modes byte native) (preprocess (pps ppx_sexp_conv ppx_deriving_yojson ppx_let ppx_deriving.show bisect_ppx --conditional))) diff --git a/src/runners/scilla_server_http.ml b/src/runners/scilla_server_http.ml new file mode 100644 index 000000000..2e502fcad --- /dev/null +++ b/src/runners/scilla_server_http.ml @@ -0,0 +1,72 @@ +open Core +open Scilla_eval +open Opium +open Yojson.Safe +open Core +open Scilla_base +open Scilla_server_lib.Api +open IPCUtil +open ErrorUtils +open Error_checking_mutex + +module M = Idl.IdM +module IDL = Idl.Make (M) +module Server = API (IDL.GenServer ()) + +let mk_handler_no_args callback () = + try IDL.ErrM.return @@ callback () + with FatalError msg -> + IDL.ErrM.return_err RPCError.{ code = 0; message = msg } + +(* Makes a handler that executes the given [callback] with [args] and returns it. **) +let mk_handler callback args = + (* Force the -jsonerrors flag *) + let args = "-jsonerrors" :: args in + try IDL.ErrM.return @@ callback (Some args) + with FatalError msg -> + IDL.ErrM.return_err RPCError.{ code = 0; message = msg } + +let server_implementation () = + let runner args = + let output, _ = Runner.run args ~exe_name:"scilla-runner" in + Yojson.Basic.pretty_to_string output + in + let disambiguator args = + Disambiguator.run args ~exe_name:"scilla-disambiguator" + in + let version () = + let major, minor, patch = Syntax.scilla_version in + Printf.sprintf "{ \"scilla_version\": \"%d.%d.%d\" }" major minor patch + in + (* Handlers *) + Server.runner @@ mk_handler runner; + Server.checker @@ mk_handler (Checker.run ~exe_name:"scilla-checker"); + Server.disambiguator @@ mk_handler disambiguator; + Server.version @@ mk_handler_no_args version; + Server.implementation + +let run_handler req = + let open Lwt.Syntax in + let+ req = Request.to_plain_text req in + let (version, id, req) = Jsonrpc.version_id_and_call_of_string req in + + let rpc = IDL.server (server_implementation ()) in + let res = + try M.run (rpc req) + with e -> + print_endline (Exn.to_string e); + Rpc.failure + (RPCError.rpc_of_t + RPCError. + { code = 0; message = "scilla-server: incorrect invocation" }) + in + let str = Jsonrpc.string_of_response ~id ~version res in + + Response.of_plain_text str +;; + +let _ = + App.empty + |> App.post "/run" (Error_checking_mutex.synchronize run_handler) + |> App.run_command +;; From 8570f5f6833ef4ed1160a37bbafa7e098dcba016 Mon Sep 17 00:00:00 2001 From: James Hinshelwood Date: Wed, 10 Apr 2024 11:35:51 +0100 Subject: [PATCH 02/17] Fix test CI --- .github/workflows/ci-image-test.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index b7ad91119..6a64e8cc8 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -18,6 +18,8 @@ jobs: contents: write name: tests runs-on: docker + env: + GCP_REGISTRY_DOMAIN: asia-docker.pkg.dev steps: - name: Clean environment # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). @@ -42,9 +44,17 @@ jobs: with: registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com - name: Build Docker images - run: | - DOCKER_BUILDKIT=1 docker build -t scilla:tests --build-arg ACCOUNT_ID=${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }} -f docker/Dockerfile.test . - shell: bash + uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 + with: + file: docker/Dockerfile + push: true + tag: scilla:tests + registry: ${{ env.GCP_REGISTRY_DOMAIN }} + build-args: | + ACCOUNT_ID=${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }} + workload-identity-provider: "${{ secrets.GCP_PRD_GITHUB_WIF }}" + service-account: "${{ secrets.GCP_STG_GITHUB_SA_DOCKER_REGISTRY }}" + cache-key: ${{ github.event.repository.name }} - name: Run make test run: | docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' From 359cc19658671f28768411574434df844979bc20 Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 13:38:20 +0200 Subject: [PATCH 03/17] wip: integration tests --- .github/workflows/ci-image-test.yml | 33 +++++++++++++---------------- docker/Dockerfile.test | 4 +--- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 6a64e8cc8..070d9b3a1 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -10,6 +10,7 @@ on: pull_request: branches: - 'master' + - 'main-zq2' jobs: run-tests: @@ -20,6 +21,7 @@ jobs: runs-on: docker env: GCP_REGISTRY_DOMAIN: asia-docker.pkg.dev + GCP_REGISTRY: asia-docker.pkg.dev/${{ secrets.GCP_PRD_REGISTRY_PROJECT_ID }}/zilliqa-private steps: - name: Clean environment # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). @@ -33,28 +35,23 @@ jobs: with: fetch-depth: 0 ref: ${{ inputs.commitOrTag }} - - name: Configure AWS Credentials - uses: Zilliqa/gh-actions-workflows/actions/configure-aws-credentials@v1 + - name: "Configure GCP Credentials" + id: google-auth + uses: "google-github-actions/auth@v1" with: - role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} - oidc-role: ${{ secrets.OIDC_ROLE }} - aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} - - name: Login to the registry + token_format: "access_token" + workload_identity_provider: "${{ secrets.GCP_PRD_GITHUB_WIF }}" + service_account: "${{ secrets.GCP_PRD_GITHUB_SA_DOCKER_REGISTRY }}" + create_credentials_file: true + - name: Login to the GCP registry uses: docker/login-action@v2 with: - registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com - - name: Build Docker images - uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 - with: - file: docker/Dockerfile - push: true - tag: scilla:tests registry: ${{ env.GCP_REGISTRY_DOMAIN }} - build-args: | - ACCOUNT_ID=${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }} - workload-identity-provider: "${{ secrets.GCP_PRD_GITHUB_WIF }}" - service-account: "${{ secrets.GCP_STG_GITHUB_SA_DOCKER_REGISTRY }}" - cache-key: ${{ github.event.repository.name }} + username: "oauth2accesstoken" + password: "${{ steps.google-auth.outputs.access_token }}" + - name: Build Docker images + run: DOCKER_BUILDKIT=1 docker build -t scilla:test -f docker/Dockerfile.test . + shell: bash - name: Run make test run: | docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index f1e533cb7..f7d66831c 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -1,6 +1,4 @@ -ARG ACCOUNT_ID - -FROM ${ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9 +FROM zilliqa/scilla:v0.13.5 ENV VCPKG_ROOT="/vcpkg" ENV SCILLA_REPO_ROOT="/scilla/0" From 4ad6cd98fba33f919d65336cab134a8e8318e24b Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 13:41:46 +0200 Subject: [PATCH 04/17] fix --- .github/workflows/ci-image-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 070d9b3a1..d93065479 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -50,7 +50,7 @@ jobs: username: "oauth2accesstoken" password: "${{ steps.google-auth.outputs.access_token }}" - name: Build Docker images - run: DOCKER_BUILDKIT=1 docker build -t scilla:test -f docker/Dockerfile.test . + run: docker build -t scilla:test -f docker/Dockerfile.test . shell: bash - name: Run make test run: | From 38bd975b547d8ab74940a36b7cf508a644a0e2ec Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 13:44:45 +0200 Subject: [PATCH 05/17] add opam installation --- docker/Dockerfile.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index f7d66831c..488a99cd6 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -7,7 +7,7 @@ WORKDIR /scilla/0/ COPY . /scilla/0/ RUN apt update \ - && apt install -y sudo + && apt install -y sudo opam RUN eval $(opam env) \ && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes From 41fb89d97a29c770eabb03f3bf39034c86bdc94a Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 13:47:52 +0200 Subject: [PATCH 06/17] add opam init --- docker/Dockerfile.test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index 488a99cd6..770caf04d 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -10,6 +10,7 @@ RUN apt update \ && apt install -y sudo opam RUN eval $(opam env) \ - && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes + && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam init --yes && \ + LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes RUN ./scripts/install_shellcheck_ubuntu.sh From 537b230bb421550e07ee48064f6fa3c030addab5 Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 13:52:04 +0200 Subject: [PATCH 07/17] fix typo --- .github/workflows/ci-image-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index d93065479..13c93f0b8 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -50,7 +50,7 @@ jobs: username: "oauth2accesstoken" password: "${{ steps.google-auth.outputs.access_token }}" - name: Build Docker images - run: docker build -t scilla:test -f docker/Dockerfile.test . + run: docker build -t scilla:tests -f docker/Dockerfile.test . shell: bash - name: Run make test run: | From 6a9b84264bb5d1e1d9b0837042c3488a3b60bd02 Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 13:59:26 +0200 Subject: [PATCH 08/17] wip --- docker/Dockerfile.test | 118 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 112 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index 770caf04d..e1a1956dd 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -1,14 +1,120 @@ -FROM zilliqa/scilla:v0.13.5 +# Common dependencies of the builder and runner stages. +FROM ubuntu:22.04 AS base +# Format guideline: one package per line and keep them alphabetically sorted +RUN apt-get update -y \ + && apt-get install -y software-properties-common \ + && apt-get update && apt-get install -y --no-install-recommends \ + autoconf \ + bison \ + build-essential \ + ca-certificates \ + ccache \ + cron \ + curl \ + dnsutils \ + gawk \ + git \ + lcov \ + libcurl4-openssl-dev \ + libev-dev \ + libgmp-dev \ + libpcre3-dev \ + libssl-dev \ + libtool \ + libxml2-utils \ + ninja-build \ + nload \ + ocaml \ + ocl-icd-opencl-dev \ + opam \ + openssh-client \ + patchelf \ + pkg-config \ + rsync \ + rsyslog \ + tar \ + trickle \ + unzip \ + vim \ + wget \ + zip \ + zlib1g-dev \ + && apt-get remove -y cmake python2 python3 && apt-get autoremove -y -ENV VCPKG_ROOT="/vcpkg" -ENV SCILLA_REPO_ROOT="/scilla/0" + CMake 3.24 +ARG CMAKE_VERSION=3.24.2 +RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh \ + && mkdir -p "${HOME}"/.local \ + && bash ./cmake-${CMAKE_VERSION}-Linux-x86_64.sh --skip-license --prefix="${HOME}"/.local/ \ + && "${HOME}"/.local/bin/cmake --version \ + && rm cmake-${CMAKE_VERSION}-Linux-x86_64.sh +ENV PATH="/root/.local/bin:${PATH}" + +# Setup ccache +RUN ln -s "$(which ccache)" /usr/local/bin/gcc \ + && ln -s "$(which ccache)" /usr/local/bin/g++ \ + && ln -s "$(which ccache)" /usr/local/bin/cc \ + && ln -s "$(which ccache)" /usr/local/bin/c++ + +# This tag must be equivalent to the hash specified by "builtin-baseline" in vcpkg.json +ARG VCPKG_COMMIT_OR_TAG=2022.09.27 +ARG VCPKG_ROOT=/vcpkg + +RUN ccache -p && ccache -z + +# If COMMIT_OR_TAG is a branch name or a tag, clone a shallow copy which is +# faster; if this fails, just clone the full repo and checkout the commit. +RUN git clone https://github.com/microsoft/vcpkg ${VCPKG_ROOT} \ + && git -C ${VCPKG_ROOT} checkout ${VCPKG_COMMIT_OR_TAG} \ + && ${VCPKG_ROOT}/bootstrap-vcpkg.sh + +RUN apt-get remove -y python3-dev && apt-get autoremove -y +# Manually input tag or commit, can be overwritten by docker build-args +ARG MAJOR_VERSION=0 +ARG SOURCE_DIR="/scilla/${MAJOR_VERSION}" + +WORKDIR ${SOURCE_DIR} + +ENV OCAML_VERSION 4.12.0 + +COPY vcpkg-registry/ vcpkg-registry +COPY vcpkg-configuration.json . +COPY vcpkg.json . +COPY vcpkg-ocaml/ vcpkg-ocaml +COPY src/ src +COPY scripts/ scripts +COPY dune* ./ +COPY Makefile . +COPY scilla.opam . +COPY shell.nix . +COPY .ocamlformat . +COPY tests/ tests + +# Make sure vcpkg installs brings in the dependencies +RUN --mount=type=cache,target=/root/.cache/vcpkg/ ${VCPKG_ROOT}/vcpkg install --triplet=x64-linux-dynamic + +RUN update-alternatives --install /usr/bin/python python "${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/tools/python3/python3.10" 10 \ + && update-alternatives --install /usr/bin/python3 python3 "${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/tools/python3/python3.10" 10 # set python3 as default instead python3 + +ENV PKG_CONFIG_PATH="${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/lib/pkgconfig" +ENV SCILLA_REPO_ROOT="${SOURCE_DIR}" + +RUN make opamdep-ci \ + && echo '. ~/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true ' >> ~/.bashrc \ + && eval $(opam env) \ + && make + +ARG BUILD_DIR="${SOURCE_DIR}/_build/default" +ARG VCPKG_INSTALL_LIB_DIR="${BUILD_DIR}/vcpkg_installed/x64-linux-dynamic/lib" + +RUN mkdir -p ${VCPKG_INSTALL_LIB_DIR} \ + && ldd ${BUILD_DIR}/src/runners/*.exe | grep vcpkg_installed | gawk '{print $3}' | xargs -I{} cp {} ${VCPKG_INSTALL_LIB_DIR} \ + && rm -rf vcpkg_installed \ + && ln -s ${BUILD_DIR}/vcpkg_installed vcpkg_installed WORKDIR /scilla/0/ COPY . /scilla/0/ -RUN apt update \ - && apt install -y sudo opam - RUN eval $(opam env) \ && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam init --yes && \ LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes From 8106f7b86be250e04ec05539120cca9cad243d9e Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 14:00:46 +0200 Subject: [PATCH 09/17] wip --- docker/Dockerfile.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index e1a1956dd..e4a54bd04 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -41,7 +41,7 @@ RUN apt-get update -y \ zlib1g-dev \ && apt-get remove -y cmake python2 python3 && apt-get autoremove -y - CMake 3.24 +# CMake 3.24 ARG CMAKE_VERSION=3.24.2 RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh \ && mkdir -p "${HOME}"/.local \ From 479fef2a3a940fda05d81b8292059cd433189f4e Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 14:05:05 +0200 Subject: [PATCH 10/17] wip --- docker/Dockerfile.test | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index e4a54bd04..5a4855188 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -90,27 +90,11 @@ COPY shell.nix . COPY .ocamlformat . COPY tests/ tests -# Make sure vcpkg installs brings in the dependencies -RUN --mount=type=cache,target=/root/.cache/vcpkg/ ${VCPKG_ROOT}/vcpkg install --triplet=x64-linux-dynamic - -RUN update-alternatives --install /usr/bin/python python "${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/tools/python3/python3.10" 10 \ - && update-alternatives --install /usr/bin/python3 python3 "${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/tools/python3/python3.10" 10 # set python3 as default instead python3 - -ENV PKG_CONFIG_PATH="${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/lib/pkgconfig" -ENV SCILLA_REPO_ROOT="${SOURCE_DIR}" - RUN make opamdep-ci \ && echo '. ~/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true ' >> ~/.bashrc \ && eval $(opam env) \ && make -ARG BUILD_DIR="${SOURCE_DIR}/_build/default" -ARG VCPKG_INSTALL_LIB_DIR="${BUILD_DIR}/vcpkg_installed/x64-linux-dynamic/lib" - -RUN mkdir -p ${VCPKG_INSTALL_LIB_DIR} \ - && ldd ${BUILD_DIR}/src/runners/*.exe | grep vcpkg_installed | gawk '{print $3}' | xargs -I{} cp {} ${VCPKG_INSTALL_LIB_DIR} \ - && rm -rf vcpkg_installed \ - && ln -s ${BUILD_DIR}/vcpkg_installed vcpkg_installed WORKDIR /scilla/0/ COPY . /scilla/0/ From e832014bbdbd829d219e22440d1123cc3e6cf34a Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 14:20:36 +0200 Subject: [PATCH 11/17] wip --- docker/Dockerfile.test | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index 5a4855188..5db3b5444 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -91,9 +91,7 @@ COPY .ocamlformat . COPY tests/ tests RUN make opamdep-ci \ - && echo '. ~/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true ' >> ~/.bashrc \ - && eval $(opam env) \ - && make + && echo '. ~/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true ' >> ~/.bashrc WORKDIR /scilla/0/ From 1f09241a586f6b67c4e8c3091b6f7a7234df5025 Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 14:39:54 +0200 Subject: [PATCH 12/17] wip --- docker/Dockerfile.test | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index 5db3b5444..6883195e4 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -1,9 +1,12 @@ +# escape=\ + # Common dependencies of the builder and runner stages. FROM ubuntu:22.04 AS base # Format guideline: one package per line and keep them alphabetically sorted RUN apt-get update -y \ && apt-get install -y software-properties-common \ && apt-get update && apt-get install -y --no-install-recommends \ + docker-buildx-plugin \ autoconf \ bison \ build-essential \ @@ -41,6 +44,8 @@ RUN apt-get update -y \ zlib1g-dev \ && apt-get remove -y cmake python2 python3 && apt-get autoremove -y +FROM base AS builder + # CMake 3.24 ARG CMAKE_VERSION=3.24.2 RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh \ @@ -90,9 +95,27 @@ COPY shell.nix . COPY .ocamlformat . COPY tests/ tests +# Make sure vcpkg installs brings in the dependencies +RUN --mount=type=cache,target=/root/.cache/vcpkg/ ${VCPKG_ROOT}/vcpkg install --triplet=x64-linux-dynamic + +RUN update-alternatives --install /usr/bin/python python "${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/tools/python3/python3.10" 10 \ + && update-alternatives --install /usr/bin/python3 python3 "${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/tools/python3/python3.10" 10 # set python3 as default instead python3 + +ENV PKG_CONFIG_PATH="${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/lib/pkgconfig" +ENV SCILLA_REPO_ROOT="${SOURCE_DIR}" + RUN make opamdep-ci \ - && echo '. ~/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true ' >> ~/.bashrc + && echo '. ~/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true ' >> ~/.bashrc \ + && eval $(opam env) \ + && make + +ARG BUILD_DIR="${SOURCE_DIR}/_build/default" +ARG VCPKG_INSTALL_LIB_DIR="${BUILD_DIR}/vcpkg_installed/x64-linux-dynamic/lib" +RUN mkdir -p ${VCPKG_INSTALL_LIB_DIR} \ + && ldd ${BUILD_DIR}/src/runners/*.exe | grep vcpkg_installed | gawk '{print $3}' | xargs -I{} cp {} ${VCPKG_INSTALL_LIB_DIR} \ + && rm -rf vcpkg_installed \ + && ln -s ${BUILD_DIR}/vcpkg_installed vcpkg_installed WORKDIR /scilla/0/ COPY . /scilla/0/ From 89f9c9320d7ea684cb1813f2508a4b21f930169a Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 14:40:06 +0200 Subject: [PATCH 13/17] wip --- .github/workflows/ci-image-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 13c93f0b8..fa1d5f015 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -27,6 +27,7 @@ jobs: # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). shell: bash run: | + docker-buildx-plugin \ docker system df docker system prune -a -f --filter "until=336h" docker system df @@ -50,7 +51,7 @@ jobs: username: "oauth2accesstoken" password: "${{ steps.google-auth.outputs.access_token }}" - name: Build Docker images - run: docker build -t scilla:tests -f docker/Dockerfile.test . + run: DOCKER_BUILDKIT=1 docker build -t scilla:tests -f docker/Dockerfile.test . shell: bash - name: Run make test run: | From bd3240c2b8788aa4d3d06164bb8b4ee9aea10be5 Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 14:41:09 +0200 Subject: [PATCH 14/17] wip --- .github/workflows/ci-image-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index fa1d5f015..9a4c07e4a 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -27,7 +27,7 @@ jobs: # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). shell: bash run: | - docker-buildx-plugin \ + apt update && apt install -y docker-buildx-plugin \ docker system df docker system prune -a -f --filter "until=336h" docker system df From db4bcb479b661a877fb327970a0b8e26802e739d Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 14:43:55 +0200 Subject: [PATCH 15/17] fix --- .github/workflows/ci-image-test.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml index 9a4c07e4a..23a3f4fca 100644 --- a/.github/workflows/ci-image-test.yml +++ b/.github/workflows/ci-image-test.yml @@ -27,7 +27,6 @@ jobs: # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). shell: bash run: | - apt update && apt install -y docker-buildx-plugin \ docker system df docker system prune -a -f --filter "until=336h" docker system df @@ -50,9 +49,11 @@ jobs: registry: ${{ env.GCP_REGISTRY_DOMAIN }} username: "oauth2accesstoken" password: "${{ steps.google-auth.outputs.access_token }}" - - name: Build Docker images - run: DOCKER_BUILDKIT=1 docker build -t scilla:tests -f docker/Dockerfile.test . - shell: bash + - uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 + with: + file: docker/Dockerfile.test + push: false + tag: cilla:tests - name: Run make test run: | docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' From 6cac17922892d52318d8afc402ad87842b1b2ff8 Mon Sep 17 00:00:00 2001 From: Mauro Medda Date: Wed, 10 Apr 2024 14:47:19 +0200 Subject: [PATCH 16/17] fix --- docker/Dockerfile.test | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test index 6883195e4..e0d2bfb0c 100644 --- a/docker/Dockerfile.test +++ b/docker/Dockerfile.test @@ -6,7 +6,6 @@ FROM ubuntu:22.04 AS base RUN apt-get update -y \ && apt-get install -y software-properties-common \ && apt-get update && apt-get install -y --no-install-recommends \ - docker-buildx-plugin \ autoconf \ bison \ build-essential \ From 9769f1398686473260526a267d37f34727665502 Mon Sep 17 00:00:00 2001 From: James Hinshelwood Date: Wed, 10 Apr 2024 18:28:40 +0100 Subject: [PATCH 17/17] Delete test CI! --- .github/workflows/ci-image-test.yml | 72 ---------------- docker/Dockerfile.test | 126 ---------------------------- 2 files changed, 198 deletions(-) delete mode 100644 .github/workflows/ci-image-test.yml delete mode 100644 docker/Dockerfile.test diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml deleted file mode 100644 index 23a3f4fca..000000000 --- a/.github/workflows/ci-image-test.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: CI - Integration tests - -on: - workflow_dispatch: - inputs: - commitOrTag: - description: 'Commit or tag' - required: false - default: '' - pull_request: - branches: - - 'master' - - 'main-zq2' - -jobs: - run-tests: - permissions: - id-token: write - contents: write - name: tests - runs-on: docker - env: - GCP_REGISTRY_DOMAIN: asia-docker.pkg.dev - GCP_REGISTRY: asia-docker.pkg.dev/${{ secrets.GCP_PRD_REGISTRY_PROJECT_ID }}/zilliqa-private - steps: - - name: Clean environment - # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). - shell: bash - run: | - docker system df - docker system prune -a -f --filter "until=336h" - docker system df - - name: 'Checkout scm ${{ inputs.commitOrTag }}' - uses: actions/checkout@v3 - with: - fetch-depth: 0 - ref: ${{ inputs.commitOrTag }} - - name: "Configure GCP Credentials" - id: google-auth - uses: "google-github-actions/auth@v1" - with: - token_format: "access_token" - workload_identity_provider: "${{ secrets.GCP_PRD_GITHUB_WIF }}" - service_account: "${{ secrets.GCP_PRD_GITHUB_SA_DOCKER_REGISTRY }}" - create_credentials_file: true - - name: Login to the GCP registry - uses: docker/login-action@v2 - with: - registry: ${{ env.GCP_REGISTRY_DOMAIN }} - username: "oauth2accesstoken" - password: "${{ steps.google-auth.outputs.access_token }}" - - uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 - with: - file: docker/Dockerfile.test - push: false - tag: cilla:tests - - name: Run make test - run: | - docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' - shell: bash - - name: Run make test_server - run: | - docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server' - shell: bash - - name: Run make coveralls - run: | - docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${{ github.run_number }}' - shell: bash - - name: Run make lint - run: | - docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint' - shell: bash diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test deleted file mode 100644 index e0d2bfb0c..000000000 --- a/docker/Dockerfile.test +++ /dev/null @@ -1,126 +0,0 @@ -# escape=\ - -# Common dependencies of the builder and runner stages. -FROM ubuntu:22.04 AS base -# Format guideline: one package per line and keep them alphabetically sorted -RUN apt-get update -y \ - && apt-get install -y software-properties-common \ - && apt-get update && apt-get install -y --no-install-recommends \ - autoconf \ - bison \ - build-essential \ - ca-certificates \ - ccache \ - cron \ - curl \ - dnsutils \ - gawk \ - git \ - lcov \ - libcurl4-openssl-dev \ - libev-dev \ - libgmp-dev \ - libpcre3-dev \ - libssl-dev \ - libtool \ - libxml2-utils \ - ninja-build \ - nload \ - ocaml \ - ocl-icd-opencl-dev \ - opam \ - openssh-client \ - patchelf \ - pkg-config \ - rsync \ - rsyslog \ - tar \ - trickle \ - unzip \ - vim \ - wget \ - zip \ - zlib1g-dev \ - && apt-get remove -y cmake python2 python3 && apt-get autoremove -y - -FROM base AS builder - -# CMake 3.24 -ARG CMAKE_VERSION=3.24.2 -RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh \ - && mkdir -p "${HOME}"/.local \ - && bash ./cmake-${CMAKE_VERSION}-Linux-x86_64.sh --skip-license --prefix="${HOME}"/.local/ \ - && "${HOME}"/.local/bin/cmake --version \ - && rm cmake-${CMAKE_VERSION}-Linux-x86_64.sh -ENV PATH="/root/.local/bin:${PATH}" - -# Setup ccache -RUN ln -s "$(which ccache)" /usr/local/bin/gcc \ - && ln -s "$(which ccache)" /usr/local/bin/g++ \ - && ln -s "$(which ccache)" /usr/local/bin/cc \ - && ln -s "$(which ccache)" /usr/local/bin/c++ - -# This tag must be equivalent to the hash specified by "builtin-baseline" in vcpkg.json -ARG VCPKG_COMMIT_OR_TAG=2022.09.27 -ARG VCPKG_ROOT=/vcpkg - -RUN ccache -p && ccache -z - -# If COMMIT_OR_TAG is a branch name or a tag, clone a shallow copy which is -# faster; if this fails, just clone the full repo and checkout the commit. -RUN git clone https://github.com/microsoft/vcpkg ${VCPKG_ROOT} \ - && git -C ${VCPKG_ROOT} checkout ${VCPKG_COMMIT_OR_TAG} \ - && ${VCPKG_ROOT}/bootstrap-vcpkg.sh - -RUN apt-get remove -y python3-dev && apt-get autoremove -y -# Manually input tag or commit, can be overwritten by docker build-args -ARG MAJOR_VERSION=0 -ARG SOURCE_DIR="/scilla/${MAJOR_VERSION}" - -WORKDIR ${SOURCE_DIR} - -ENV OCAML_VERSION 4.12.0 - -COPY vcpkg-registry/ vcpkg-registry -COPY vcpkg-configuration.json . -COPY vcpkg.json . -COPY vcpkg-ocaml/ vcpkg-ocaml -COPY src/ src -COPY scripts/ scripts -COPY dune* ./ -COPY Makefile . -COPY scilla.opam . -COPY shell.nix . -COPY .ocamlformat . -COPY tests/ tests - -# Make sure vcpkg installs brings in the dependencies -RUN --mount=type=cache,target=/root/.cache/vcpkg/ ${VCPKG_ROOT}/vcpkg install --triplet=x64-linux-dynamic - -RUN update-alternatives --install /usr/bin/python python "${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/tools/python3/python3.10" 10 \ - && update-alternatives --install /usr/bin/python3 python3 "${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/tools/python3/python3.10" 10 # set python3 as default instead python3 - -ENV PKG_CONFIG_PATH="${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/lib/pkgconfig" -ENV SCILLA_REPO_ROOT="${SOURCE_DIR}" - -RUN make opamdep-ci \ - && echo '. ~/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true ' >> ~/.bashrc \ - && eval $(opam env) \ - && make - -ARG BUILD_DIR="${SOURCE_DIR}/_build/default" -ARG VCPKG_INSTALL_LIB_DIR="${BUILD_DIR}/vcpkg_installed/x64-linux-dynamic/lib" - -RUN mkdir -p ${VCPKG_INSTALL_LIB_DIR} \ - && ldd ${BUILD_DIR}/src/runners/*.exe | grep vcpkg_installed | gawk '{print $3}' | xargs -I{} cp {} ${VCPKG_INSTALL_LIB_DIR} \ - && rm -rf vcpkg_installed \ - && ln -s ${BUILD_DIR}/vcpkg_installed vcpkg_installed - -WORKDIR /scilla/0/ -COPY . /scilla/0/ - -RUN eval $(opam env) \ - && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam init --yes && \ - LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes - -RUN ./scripts/install_shellcheck_ubuntu.sh