From 61b6c668f72de9f3da24aeffdecf73629c9b1cbe Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 25 Jul 2023 04:17:01 +0000 Subject: [PATCH 01/90] Add network id ivar --- src/lib/integration_test_lib/intf.ml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index 6e44f82783f..5977f4f4c28 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -103,6 +103,8 @@ module Engine = struct val genesis_keypairs : t -> Network_keypair.t Core.String.Map.t val initialize_infra : logger:Logger.t -> t -> unit Malleable_error.t + + val id : string Async.Ivar.t end module type Network_manager_intf = sig From c8076db11f32394deb45188ab92a3a0f565d292e Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 25 Jul 2023 04:19:28 +0000 Subject: [PATCH 02/90] Add abstract engine skeleton --- src/dune-project | 1 + .../cli_inputs.ml | 55 +++ src/lib/integration_test_abstract_engine/dune | 58 +++ .../graphql_polling_log_engine.ml | 141 ++++++ .../graphql_polling_log_engine.mli | 3 + .../integration_test_abstract_engine.ml | 6 + .../integration_test_abstract_engine.mli | 1 + .../mina_automation.ml | 419 ++++++++++++++++++ .../terraform.ml | 111 +++++ 9 files changed, 795 insertions(+) create mode 100644 src/lib/integration_test_abstract_engine/cli_inputs.ml create mode 100644 src/lib/integration_test_abstract_engine/dune create mode 100644 src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml create mode 100644 src/lib/integration_test_abstract_engine/graphql_polling_log_engine.mli create mode 100644 src/lib/integration_test_abstract_engine/integration_test_abstract_engine.ml create mode 100644 src/lib/integration_test_abstract_engine/integration_test_abstract_engine.mli create mode 100644 src/lib/integration_test_abstract_engine/mina_automation.ml create mode 100644 src/lib/integration_test_abstract_engine/terraform.ml diff --git a/src/dune-project b/src/dune-project index 5969c24462c..98183a75f44 100644 --- a/src/dune-project +++ b/src/dune-project @@ -66,6 +66,7 @@ (package (name immutable_array)) (package (name inline_test_quiet_logs)) (package (name integers_stubs_js)) +(package (name integration_test_abstract_engine)) (package (name integration_test_cloud_engine)) (package (name integration_test_lib)) (package (name internal_tracing)) diff --git a/src/lib/integration_test_abstract_engine/cli_inputs.ml b/src/lib/integration_test_abstract_engine/cli_inputs.ml new file mode 100644 index 00000000000..7bcc7a257aa --- /dev/null +++ b/src/lib/integration_test_abstract_engine/cli_inputs.ml @@ -0,0 +1,55 @@ +open Cmdliner + +type t = + { mina_automation_location : string + ; check_capacity : bool + ; check_capacity_delay : int + ; check_capacity_retries : int + } + +let term = + let mina_automation_location = + let doc = + "Location of the Mina automation repository to use when deploying the \ + network." + in + let env = Arg.env_var "MINA_AUTOMATION_LOCATION" ~doc in + Arg.( + value & opt string "./automation" + & info + [ "mina-automation-location" ] + ~env ~docv:"MINA_AUTOMATION_LOCATION" ~doc) + in + let check_capacity = + let doc = + "Whether or not to check the capacity of the cloud cluster before \ + execution. Default: true" + in + Arg.(value & opt bool true & info [ "capacity-check" ] ~doc) + in + let check_capacity_delay = + let doc = + "Upon a failed capacity check, how much time in seconds to wait before \ + trying. Only holds meaning if check-capacity is true. Default: 60" + in + Arg.(value & opt int 60 & info [ "capacity-check-delay" ] ~doc) + in + let check_capacity_retries = + let doc = + "Upon a failed capacity check, how many times to retry before giving \ + up. Only holds meaning if check-capacity is true. Default: 10" + in + Arg.(value & opt int 10 & info [ "capacity-check-retries" ] ~doc) + in + let cons_inputs mina_automation_location check_capacity check_capacity_delay + check_capacity_retries = + { mina_automation_location + ; check_capacity + ; check_capacity_delay + ; check_capacity_retries + } + in + + Term.( + const cons_inputs $ mina_automation_location $ check_capacity + $ check_capacity_delay $ check_capacity_retries) diff --git a/src/lib/integration_test_abstract_engine/dune b/src/lib/integration_test_abstract_engine/dune new file mode 100644 index 00000000000..64fd802203a --- /dev/null +++ b/src/lib/integration_test_abstract_engine/dune @@ -0,0 +1,58 @@ +(library + (public_name integration_test_abstract_engine) + (name integration_test_abstract_engine) + (inline_tests) + (instrumentation (backend bisect_ppx)) + (preprocessor_deps ../../graphql-ppx-config.inc ../../../graphql_schema.json) + (preprocess (pps + ppx_mina ppx_version ppx_let ppx_pipebang ppx_inline_test + ppx_custom_printf ppx_deriving_yojson lens.ppx_deriving + ppx_sexp_conv ppx_assert ppx_deriving.eq + graphql_ppx -- %{read-lines:../../graphql-ppx-config.inc})) +(libraries + ;; opam libraries + async_unix + async_kernel + core_kernel + core + async + cmdliner + base + uri + sexplib0 + stdio + result + base.caml + integers + re2 + ;; local libraries + key_gen + integration_test_lib + graphql_lib + mina_runtime_config + mina_base + genesis_constants + genesis_ledger_helper + logger + mina_base_import + signature_lib + currency + mina_version + timeout_lib + mina_numbers + mina_stdlib + mina_transaction + file_system + pickles + pickles_types + backend + kimchi_pasta + kimchi_backend.pasta.basic + with_hash + data_hash_lib + generated_graphql_queries + mina_graphql + error_json + ppx_inline_test.config + ) +) diff --git a/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml b/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml new file mode 100644 index 00000000000..a2d2a402cfb --- /dev/null +++ b/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml @@ -0,0 +1,141 @@ +open Async +open Core +open Integration_test_lib +module Timeout = Timeout_lib.Core_time +module Node = Abstract_network.Node + +(** This implements Log_engine_intf for integration tests, by creating a simple system that polls a mina daemon's graphql endpoint for fetching logs*) + +let log_filter_of_event_type ev_existential = + let open Event_type in + let (Event_type ev_type) = ev_existential in + let (module Ty) = event_type_module ev_type in + match Ty.parse with + | From_error_log _ -> + [] (* TODO: Do we need this? *) + | From_daemon_log (struct_id, _) -> + [ Structured_log_events.string_of_id struct_id ] + | From_puppeteer_log _ -> + [] +(* TODO: Do we need this? *) + +let all_event_types_log_filter = + List.bind ~f:log_filter_of_event_type Event_type.all_event_types + +type t = + { logger : Logger.t + ; event_writer : (Node.t * Event_type.event) Pipe.Writer.t + ; event_reader : (Node.t * Event_type.event) Pipe.Reader.t + ; background_job : unit Deferred.Or_error.t + } + +let event_reader { event_reader; _ } = event_reader + +let parse_event_from_log_entry ~logger log_entry = + let open Or_error.Let_syntax in + let open Json_parsing in + Or_error.try_with_join (fun () -> + let payload = Yojson.Safe.from_string log_entry in + let%map event = + let%bind msg = + parse (parser_from_of_yojson Logger.Message.of_yojson) payload + in + let event_id = + Option.map ~f:Structured_log_events.string_of_id msg.event_id + in + [%log spam] "parsing daemon structured event, event_id = $event_id" + ~metadata:[ ("event_id", [%to_yojson: string option] event_id) ] ; + match msg.event_id with + | Some _ -> + Event_type.parse_daemon_event msg + | None -> + (* Currently unreachable, but we could include error logs here if + desired. + *) + Event_type.parse_error_log msg + in + event ) + +let rec poll_get_filtered_log_entries_node ~logger ~event_writer + ~last_log_index_seen node = + let open Deferred.Let_syntax in + if not (Pipe.is_closed event_writer) then ( + let%bind () = after (Time.Span.of_ms 10000.0) in + match%bind + Abstract_network.Node.get_filtered_log_entries ~last_log_index_seen node + with + | Ok log_entries -> + Array.iter log_entries ~f:(fun log_entry -> + match parse_event_from_log_entry ~logger log_entry with + | Ok a -> + Pipe.write_without_pushback_if_open event_writer (node, a) + | Error e -> + [%log warn] "Error parsing log $error" + ~metadata:[ ("error", `String (Error.to_string_hum e)) ] ) ; + let last_log_index_seen = + Array.length log_entries + last_log_index_seen + in + poll_get_filtered_log_entries_node ~logger ~event_writer + ~last_log_index_seen node + | Error err -> + [%log error] "Encountered an error while polling $node for logs: $err" + ~metadata: + [ ("node", `String node.app_id) + ; ("err", Error_json.error_to_yojson err) + ] ; + (* Declare the node to be offline. *) + Pipe.write_without_pushback_if_open event_writer + (node, Event (Node_offline, ())) ; + (* Don't keep looping, the node may be restarting. *) + return (Ok ()) ) + else Deferred.Or_error.error_string "Event writer closed" + +let rec poll_start_filtered_log_node ~logger ~log_filter ~event_writer node = + let open Deferred.Let_syntax in + if not (Pipe.is_closed event_writer) then + match%bind + Abstract_network.Node.start_filtered_log ~logger ~log_filter node + with + | Ok () -> + return (Ok ()) + | Error _ -> + poll_start_filtered_log_node ~logger ~log_filter ~event_writer node + else Deferred.Or_error.error_string "Event writer closed" + +let rec poll_node_for_logs_in_background ~log_filter ~logger ~event_writer + (node : Node.t) = + let open Deferred.Or_error.Let_syntax in + [%log info] "Requesting for $node to start its filtered logs" + ~metadata:[ ("node", `String node.app_id) ] ; + let%bind () = + poll_start_filtered_log_node ~logger ~log_filter ~event_writer node + in + [%log info] "$node has started its filtered logs. Beginning polling" + ~metadata:[ ("node", `String node.app_id) ] ; + let%bind () = + poll_get_filtered_log_entries_node ~last_log_index_seen:0 ~logger + ~event_writer node + in + poll_node_for_logs_in_background ~log_filter ~logger ~event_writer node + +let poll_for_logs_in_background ~log_filter ~logger ~network ~event_writer = + Abstract_network.all_pods network + |> Core.String.Map.data + |> Deferred.Or_error.List.iter ~how:`Parallel + ~f:(poll_node_for_logs_in_background ~log_filter ~logger ~event_writer) + +let create ~logger ~(network : Abstract_network.t) = + let open Deferred.Or_error.Let_syntax in + let log_filter = all_event_types_log_filter in + let event_reader, event_writer = Pipe.create () in + let background_job = + poll_for_logs_in_background ~log_filter ~logger ~network ~event_writer + in + return { logger; event_reader; event_writer; background_job } + +let destroy t : unit Deferred.Or_error.t = + let open Deferred.Or_error.Let_syntax in + let { logger; event_reader = _; event_writer; background_job = _ } = t in + Pipe.close event_writer ; + [%log debug] "graphql polling log engine destroyed" ; + return () diff --git a/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.mli b/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.mli new file mode 100644 index 00000000000..e04c80e6b90 --- /dev/null +++ b/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.mli @@ -0,0 +1,3 @@ +include + Integration_test_lib.Intf.Engine.Log_engine_intf + with module Network := Abstract_network diff --git a/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.ml b/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.ml new file mode 100644 index 00000000000..37e989045fc --- /dev/null +++ b/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.ml @@ -0,0 +1,6 @@ +let name = "abstract" + +module Network = Abstract_network +module Network_config = Ci_interaction.Network_config +module Network_manager = Mina_automation.Network_manager +module Log_engine = Graphql_polling_log_engine diff --git a/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.mli b/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.mli new file mode 100644 index 00000000000..e1ba61b4e77 --- /dev/null +++ b/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.mli @@ -0,0 +1 @@ +include Integration_test_lib.Intf.Engine.S diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml new file mode 100644 index 00000000000..2a082479dc2 --- /dev/null +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -0,0 +1,419 @@ +open Core +open Async +open Integration_test_lib +open Ci_interaction + +module Network_manager = struct + type t = + { logger : Logger.t + ; testnet_name : string + ; cluster : string + ; namespace : string + ; graphql_enabled : bool + ; testnet_dir : string + ; testnet_log_filter : string + ; constants : Test_config.constants + ; seed_workloads : Abstract_network.Workload_to_deploy.t Core.String.Map.t + ; block_producer_workloads : + Abstract_network.Workload_to_deploy.t Core.String.Map.t + ; snark_coordinator_workloads : + Abstract_network.Workload_to_deploy.t Core.String.Map.t + ; snark_worker_workloads : + Abstract_network.Workload_to_deploy.t Core.String.Map.t + ; archive_workloads : + Abstract_network.Workload_to_deploy.t Core.String.Map.t + ; workloads_by_id : Abstract_network.Workload_to_deploy.t Core.String.Map.t + ; mutable deployed : bool + ; genesis_keypairs : Network_keypair.t Core.String.Map.t + } + + let run_cmd t prog args = Util.run_cmd t.testnet_dir prog args + + let run_cmd_exn t prog args = Util.run_cmd_exn t.testnet_dir prog args + + let run_cmd_or_hard_error t prog args = + Util.run_cmd_or_hard_error t.testnet_dir prog args + + let rec check_kube_capacity t ~logger ~(retries : int) ~(delay : float) : + unit Malleable_error.t = + let open Malleable_error.Let_syntax in + let%bind () = + Malleable_error.return ([%log info] "Running capacity check") + in + let%bind kubectl_top_nodes_output = + Util.run_cmd_or_hard_error "/" "kubectl" + [ "top"; "nodes"; "--sort-by=cpu"; "--no-headers" ] + in + let num_kube_nodes = + String.split_on_chars kubectl_top_nodes_output ~on:[ '\n' ] |> List.length + in + let%bind gcloud_descr_output = + Util.run_cmd_or_hard_error "/" "gcloud" + [ "container" + ; "clusters" + ; "describe" + ; cluster_name + ; "--project" + ; "o1labs-192920" + ; "--region" + ; cluster_region + ] + in + (* gcloud container clusters describe mina-integration-west1 --project o1labs-192920 --region us-west1 + this command gives us lots of information, including the max number of nodes per node pool. + *) + let%bind max_node_count_str = + Util.run_cmd_or_hard_error "/" "bash" + [ "-c" + ; Format.sprintf "echo \"%s\" | grep \"maxNodeCount\" " + gcloud_descr_output + ] + in + let max_node_count_by_node_pool = + Re2.find_all_exn (Re2.of_string "[0-9]+") max_node_count_str + |> List.map ~f:(fun str -> Int.of_string str) + in + (* We can have any number of node_pools. this string parsing will yield a list of ints, each int represents the + max_node_count for each node pool *) + let max_nodes = + List.fold max_node_count_by_node_pool ~init:0 ~f:(fun accum max_nodes -> + accum + (max_nodes * 3) ) + (* + the max_node_count_by_node_pool is per zone. us-west1 has 3 zones (we assume this never changes). + therefore to get the actual number of nodes a node_pool has, we multiply by 3. + then we sum up the number of nodes in all our node_pools to get the actual total maximum number of nodes that we can scale up to *) + in + let nodes_available = max_nodes - num_kube_nodes in + let cpus_needed_estimate = + 6 + * ( Core.Map.length t.seed_workloads + + Core.Map.length t.block_producer_workloads + + Core.Map.length t.snark_coordinator_workloads ) + (* as of 2022/07, the seed, bps, and the snark coordinator use 6 cpus. this is just a rough heuristic so we're not bothering to calculate memory needed *) + in + let cluster_nodes_needed = + Int.of_float + (Float.round_up (Float.( / ) (Float.of_int cpus_needed_estimate) 64.0)) + (* assuming that each node on the cluster has 64 cpus, as we've configured it to be in GCP as of *) + in + if nodes_available >= cluster_nodes_needed then + let%bind () = + Malleable_error.return + ([%log info] + "Capacity check passed. %d nodes are provisioned, the cluster \ + can scale up to a max of %d nodes. This test needs at least 1 \ + node to be unprovisioned." + num_kube_nodes max_nodes ) + in + Malleable_error.return () + else if retries <= 0 then + let%bind () = + Malleable_error.return + ([%log info] + "Capacity check failed. %d nodes are provisioned, the cluster \ + can scale up to a max of %d nodes. This test needs at least 1 \ + node to be unprovisioned. no more retries, thus exiting" + num_kube_nodes max_nodes ) + in + exit 7 + else + let%bind () = + Malleable_error.return + ([%log info] + "Capacity check failed. %d nodes are provisioned, the cluster \ + can scale up to a max of %d nodes. This test needs at least 1 \ + node to be unprovisioned. sleeping for 60 seconds before \ + retrying. will retry %d more times" + num_kube_nodes max_nodes (retries - 1) ) + in + let%bind () = Malleable_error.return (Thread.delay delay) in + check_kube_capacity t ~logger ~retries:(retries - 1) ~delay + + let create ~logger (network_config : Network_config.t) = + let open Malleable_error.Let_syntax in + let%bind current_cluster = + Util.run_cmd_or_hard_error "/" "kubectl" [ "config"; "current-context" ] + in + [%log info] "Using cluster: %s" current_cluster ; + let%bind all_namespaces_str = + Util.run_cmd_or_hard_error "/" "kubectl" + [ "get"; "namespaces"; "-ojsonpath={.items[*].metadata.name}" ] + in + let all_namespaces = String.split ~on:' ' all_namespaces_str in + let%bind () = + if + List.mem all_namespaces network_config.terraform.testnet_name + ~equal:String.equal + then + let%bind () = + if network_config.debug_arg then + Deferred.bind ~f:Malleable_error.return + (Util.prompt_continue + "Existing namespace of same name detected, pausing startup. \ + Enter [y/Y] to continue on and remove existing namespace, \ + start clean, and run the test; press Cntrl-C to quit out: " ) + else + Malleable_error.return + ([%log info] + "Existing namespace of same name detected; removing to start \ + clean" ) + in + Util.run_cmd_or_hard_error "/" "kubectl" + [ "delete"; "namespace"; network_config.terraform.testnet_name ] + >>| Fn.const () + else return () + in + (* TODO: prebuild genesis proof and ledger *) + let testnet_log_filter = Network_config.testnet_log_filter network_config in + (* we currently only deploy 1 seed and coordinator per deploy (will be configurable later) *) + (* seed node keyname and workload name hardcoded as "seed" *) + let seed_workloads = + Core.String.Map.add_exn Core.String.Map.empty ~key:"seed" + ~data: + (Abstract_network.Workload_to_deploy.construct_workload "seed" + (Abstract_network.Workload_to_deploy.cons_pod_info "mina") ) + in + + let snark_coordinator_workloads, snark_worker_workloads = + match network_config.terraform.snark_coordinator_config with + | Some config -> + let snark_coordinator_workloads = + if config.worker_nodes > 0 then + Core.String.Map.of_alist_exn + [ ( config.name + , Abstract_network.Workload_to_deploy.construct_workload + config.name + (Abstract_network.Workload_to_deploy.cons_pod_info "mina") + ) + ] + else Core.String.Map.of_alist_exn [] + in + let snark_worker_workloads = + if config.worker_nodes > 0 then + Core.String.Map.of_alist_exn + [ ( config.name ^ "-worker" + , Abstract_network.Workload_to_deploy.construct_workload + (config.name ^ "-worker") + (Abstract_network.Workload_to_deploy.cons_pod_info + "worker" ) ) + ] + else Core.String.Map.of_alist_exn [] + in + (snark_coordinator_workloads, snark_worker_workloads) + | None -> + (Core.String.Map.of_alist_exn [], Core.String.Map.of_alist_exn []) + in + (* + let snark_coordinator_id = + String.lowercase + (String.sub network_config.terraform.snark_worker_public_key + ~pos: + (String.length network_config.terraform.snark_worker_public_key - 6) + ~len:6 ) + in + let snark_coordinator_workloads = + if network_config.terraform.snark_worker_replicas > 0 then + [ Abstract_network.Workload_to_deploy.construct_workload + ("snark-coordinator-" ^ snark_coordinator_id) + [ Abstract_network.Workload_to_deploy.cons_pod_info "mina" ] + ] + else [] + in + let snark_worker_workloads = + if network_config.terraform.snark_worker_replicas > 0 then + [ Abstract_network.Workload_to_deploy.construct_workload + ("snark-worker-" ^ snark_coordinator_id) + (List.init network_config.terraform.snark_worker_replicas + ~f:(fun _i -> + Abstract_network.Workload_to_deploy.cons_pod_info "worker" ) + ) + ] + else [] + in *) + let block_producer_workloads = + List.map network_config.terraform.block_producer_configs + ~f:(fun bp_config -> + ( bp_config.name + , Abstract_network.Workload_to_deploy.construct_workload + bp_config.name + (Abstract_network.Workload_to_deploy.cons_pod_info + ~network_keypair:bp_config.keypair "mina" ) ) ) + |> Core.String.Map.of_alist_exn + in + let archive_workloads = + List.init network_config.terraform.archive_node_count ~f:(fun i -> + ( sprintf "archive-%d" (i + 1) + , Abstract_network.Workload_to_deploy.construct_workload + (sprintf "archive-%d" (i + 1)) + (Abstract_network.Workload_to_deploy.cons_pod_info + ~has_archive_container:true "mina" ) ) ) + |> Core.String.Map.of_alist_exn + in + let workloads_by_id = + let all_workloads = + Core.String.Map.data seed_workloads + @ Core.String.Map.data snark_coordinator_workloads + @ Core.String.Map.data snark_worker_workloads + @ Core.String.Map.data block_producer_workloads + @ Core.String.Map.data archive_workloads + in + all_workloads + |> List.map ~f:(fun w -> (w.workload_id, w)) + |> String.Map.of_alist_exn + in + let testnet_dir = + network_config.mina_automation_location ^/ "terraform/testnets" + ^/ network_config.terraform.testnet_name + in + let t = + { logger + ; cluster = cluster_id + ; namespace = network_config.terraform.testnet_name + ; testnet_name = network_config.terraform.testnet_name + ; graphql_enabled = network_config.terraform.deploy_graphql_ingress + ; testnet_dir + ; testnet_log_filter + ; constants = network_config.constants + ; seed_workloads + ; block_producer_workloads + ; snark_coordinator_workloads + ; snark_worker_workloads + ; archive_workloads + ; workloads_by_id + ; deployed = false + ; genesis_keypairs = network_config.genesis_keypairs + } + in + (* check capacity *) + let%bind () = + if network_config.check_capacity then + check_kube_capacity t ~logger + ~delay:(Float.of_int network_config.check_capacity_delay) + ~retries:network_config.check_capacity_retries + else Malleable_error.return () + in + (* making the main.tf.json *) + let open Deferred.Let_syntax in + let%bind () = + if%bind File_system.dir_exists testnet_dir then ( + [%log info] "Old terraform directory found; removing to start clean" ; + File_system.remove_dir testnet_dir ) + else return () + in + [%log info] "Making testnet dir %s" testnet_dir ; + let%bind () = Unix.mkdir testnet_dir in + let tf_filename = testnet_dir ^/ "main.tf.json" in + [%log info] "Writing network configuration into %s" tf_filename ; + Out_channel.with_file ~fail_if_exists:true tf_filename ~f:(fun ch -> + Network_config.to_terraform network_config + |> Terraform.to_string + |> Out_channel.output_string ch ) ; + [%log info] + "Writing out the genesis keys (in case you want to use them manually) to \ + testnet dir %s" + testnet_dir ; + let kps_base_path = String.concat [ testnet_dir; "/genesis_keys" ] in + let%bind () = Unix.mkdir kps_base_path in + let%bind () = + Core.String.Map.iter network_config.genesis_keypairs ~f:(fun kp -> + Network_keypair.to_yojson kp + |> Yojson.Safe.to_file + (String.concat [ kps_base_path; "/"; kp.keypair_name; ".json" ]) ) + |> Deferred.return + in + [%log info] "Initializing terraform" ; + let open Malleable_error.Let_syntax in + let%bind (_ : string) = run_cmd_or_hard_error t "terraform" [ "init" ] in + let%map (_ : string) = run_cmd_or_hard_error t "terraform" [ "validate" ] in + t + + let deploy t = + let open Malleable_error.Let_syntax in + let logger = t.logger in + if t.deployed then failwith "network already deployed" ; + [%log info] "Deploying network" ; + let%bind (_ : string) = + run_cmd_or_hard_error t "terraform" [ "apply"; "-auto-approve" ] + in + t.deployed <- true ; + let config : Abstract_network.config = + { testnet_name = t.testnet_name + ; cluster = t.cluster + ; namespace = t.namespace + ; graphql_enabled = t.graphql_enabled + ; access_token = "access_token" (* TODO: *) + } + in + let func_for_fold ~(key : string) ~data accum_M = + let%bind mp = accum_M in + let%map node = + Abstract_network.Workload_to_deploy.get_nodes_from_workload data ~config + in + Core.String.Map.add_exn mp ~key ~data:node + in + let%map seeds = + Core.String.Map.fold t.seed_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold + and block_producers = + Core.String.Map.fold t.block_producer_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold + and snark_coordinators = + Core.String.Map.fold t.snark_coordinator_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold + and snark_workers = + Core.String.Map.fold t.snark_worker_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold + and archive_nodes = + Core.String.Map.fold t.archive_workloads + ~init:(Malleable_error.return Core.String.Map.empty) + ~f:func_for_fold + in + let network = + { Abstract_network.namespace = t.namespace + ; constants = t.constants + ; seeds + ; block_producers + ; snark_coordinators + ; snark_workers + ; archive_nodes (* ; all_nodes *) + ; testnet_log_filter = t.testnet_log_filter + ; genesis_keypairs = t.genesis_keypairs + } + in + let nodes_to_string = + Fn.compose (String.concat ~sep:", ") + (List.map ~f:Abstract_network.Node.id) + in + [%log info] "Network deployed" ; + [%log info] "testnet namespace: %s" t.namespace ; + [%log info] "snark coordinators: %s" + (nodes_to_string (Core.String.Map.data network.snark_coordinators)) ; + [%log info] "snark workers: %s" + (nodes_to_string (Core.String.Map.data network.snark_workers)) ; + [%log info] "block producers: %s" + (nodes_to_string (Core.String.Map.data network.block_producers)) ; + [%log info] "archive nodes: %s" + (nodes_to_string (Core.String.Map.data network.archive_nodes)) ; + network + + let destroy t = + [%log' info t.logger] "Destroying network" ; + if not t.deployed then failwith "network not deployed" ; + let%bind _ = run_cmd_exn t "terraform" [ "destroy"; "-auto-approve" ] in + t.deployed <- false ; + Deferred.unit + + let cleanup t = + let%bind () = if t.deployed then destroy t else return () in + [%log' info t.logger] "Cleaning up network configuration" ; + let%bind () = File_system.remove_dir t.testnet_dir in + Deferred.unit + + let destroy t = + Deferred.Or_error.try_with (fun () -> destroy t) + |> Deferred.bind ~f:Malleable_error.or_hard_error +end diff --git a/src/lib/integration_test_abstract_engine/terraform.ml b/src/lib/integration_test_abstract_engine/terraform.ml new file mode 100644 index 00000000000..02eea743983 --- /dev/null +++ b/src/lib/integration_test_abstract_engine/terraform.ml @@ -0,0 +1,111 @@ +open Core + +(* this is only a partial (minimum required) implementation of the terraform json spec *) +let cons (type a) (key : string) (body : a) (to_yojson : a -> Yojson.Safe.t) = + `Assoc [ (key, to_yojson body) ] + +module Backend = struct + module Local = struct + type t = { path : string } [@@deriving to_yojson] + end + + type t = Local of Local.t + + let to_yojson = function Local x -> cons "local" x Local.to_yojson +end + +module Block = struct + module Terraform = struct + type t = { required_version : string; backend : Backend.t } + [@@deriving to_yojson] + end + + (* should probably leave these untyped, but this covers basic keys for both google and aws *) + module Provider = struct + type t = + { provider : string + ; region : string + ; alias : string option + ; project : string option + ; zone : string option + } + [@@deriving to_yojson] + + let to_yojson { provider; region; alias; project; zone } = + cons provider () (fun () -> + let open Option.Let_syntax in + let field k v = (k, `String v) in + let fields = + [ Some (field "region" region) + ; alias >>| field "alias" + ; project >>| field "project" + ; zone >>| field "zone" + ] + in + `Assoc (List.filter_map fields ~f:Fn.id) ) + end + + module Module = struct + type t = + { local_name : string + ; providers : (string * string) list + ; source : string + ; args : (string * Yojson.Safe.t) list + } + + let to_yojson { local_name; providers; source; args } = + cons local_name () (fun () -> + let const_fields = + [ ( "providers" + , `Assoc (List.map providers ~f:(fun (k, v) -> (k, `String v))) ) + ; ("source", `String source) + ] + in + `Assoc (const_fields @ args) ) + end + + module Data = struct + type t = + { data_source : string + ; local_name : string + ; args : (string * Yojson.Safe.t) list + } + + let to_yojson { data_source; local_name; args } = + cons data_source () (fun () -> cons local_name () (fun () -> `Assoc args)) + end + + module Resource = struct + type t = + { type_ : string + ; local_name : string + ; args : (string * Yojson.Safe.t) list + } + + let to_yojson { type_; local_name; args } = + cons type_ () (fun () -> cons local_name () (fun () -> `Assoc args)) + end + + type t = + | Terraform of Terraform.t + | Provider of Provider.t + | Module of Module.t + | Data of Data.t + | Resource of Resource.t + + let to_yojson = function + | Terraform x -> + cons "terraform" x Terraform.to_yojson + | Provider x -> + cons "provider" x Provider.to_yojson + | Module x -> + cons "module" x Module.to_yojson + | Data x -> + cons "data" x Data.to_yojson + | Resource x -> + cons "resource" x Resource.to_yojson +end + +type t = Block.t list [@@deriving to_yojson] + +let to_string = Fn.compose Yojson.Safe.pretty_to_string to_yojson From ce2d88828e33ca1b6cc2aa8da93d948b81e1f39a Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 25 Jul 2023 04:20:45 +0000 Subject: [PATCH 03/90] Add ci interactions for abstract engine --- .../ci_interaction.ml | 554 ++++++++++++++++++ 1 file changed, 554 insertions(+) create mode 100644 src/lib/integration_test_abstract_engine/ci_interaction.ml diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml new file mode 100644 index 00000000000..7ce324a9027 --- /dev/null +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -0,0 +1,554 @@ +open Core_kernel +open Async +open Currency +open Signature_lib +open Mina_base +open Integration_test_lib + +let aws_region = "us-west-2" + +let aws_route53_zone_id = "ZJPR9NA6W9M7F" + +let project_id = "o1labs-192920" + +let cluster_id = "gke_o1labs-192920_us-west1_mina-integration-west1" + +let cluster_name = "mina-integration-west1" + +let cluster_region = "us-west1" + +let cluster_zone = "us-west1a" + +module Network_config = struct + module Cli_inputs = Cli_inputs + + type block_producer_config = + { name : string; keypair : Network_keypair.t; libp2p_secret : string } + [@@deriving to_yojson] + + type snark_coordinator_config = + { name : string; public_key : string; worker_nodes : int } + [@@deriving to_yojson] + + type terraform_config = + { k8s_context : string + ; cluster_name : string + ; cluster_region : string + ; aws_route53_zone_id : string + ; testnet_name : string + ; deploy_graphql_ingress : bool + ; mina_image : string + ; mina_agent_image : string + ; mina_bots_image : string + ; mina_points_image : string + ; mina_archive_image : string + (* this field needs to be sent as a string to terraform, even though it's a json encoded value *) + ; runtime_config : Yojson.Safe.t + [@to_yojson fun j -> `String (Yojson.Safe.to_string j)] + ; block_producer_configs : block_producer_config list + ; log_precomputed_blocks : bool + ; archive_node_count : int + ; mina_archive_schema : string + ; mina_archive_schema_aux_files : string list + ; snark_coordinator_config : snark_coordinator_config option + ; snark_worker_fee : string + ; cpu_request : int + ; mem_request : string + ; worker_cpu_request : int + ; worker_mem_request : string + } + [@@deriving to_yojson] + + type t = + { mina_automation_location : string + ; debug_arg : bool + ; check_capacity : bool + ; check_capacity_delay : int + ; check_capacity_retries : int + ; genesis_keypairs : + (Network_keypair.t Core.String.Map.t + [@to_yojson + fun map -> + `Assoc + (Core.Map.fold_right ~init:[] + ~f:(fun ~key:k ~data:v accum -> + (k, Network_keypair.to_yojson v) :: accum ) + map )] ) + ; constants : Test_config.constants + ; terraform : terraform_config + } + [@@deriving to_yojson] + + + let terraform_config_to_assoc t = + let[@warning "-8"] (`Assoc assoc : Yojson.Safe.t) = + terraform_config_to_yojson t + in + assoc + + let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) + ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) + = + let { requires_graphql + ; genesis_ledger + ; block_producers + ; snark_coordinator + ; snark_worker_fee + ; num_archive_nodes + ; log_precomputed_blocks + ; proof_config + ; Test_config.k + ; delta + ; slots_per_epoch + ; slots_per_sub_window + ; txpool_max_size + } = + test_config + in + let user_from_env = Option.value (Unix.getenv "USER") ~default:"auto" in + let user_sanitized = + Str.global_replace (Str.regexp "\\W|_-") "" user_from_env + in + let user_len = Int.min 5 (String.length user_sanitized) in + let user = String.sub user_sanitized ~pos:0 ~len:user_len in + let git_commit = Mina_version.commit_id_short in + (* see ./src/app/test_executive/README.md for information regarding the namespace name format and length restrictions *) + let testnet_name = "it-" ^ user ^ "-" ^ git_commit ^ "-" ^ test_name in + (* check to make sure the test writer hasn't accidentally created duplicate names of accounts and keys *) + let key_names_list = + List.map genesis_ledger ~f:(fun acct -> acct.account_name) + in + if List.contains_dup ~compare:String.compare key_names_list then + failwith + "All accounts in genesis ledger must have unique names. Check to make \ + sure you are not using the same account_name more than once" ; + let all_nodes_names_list = + List.map block_producers ~f:(fun acct -> acct.node_name) + @ match snark_coordinator with None -> [] | Some n -> [ n.node_name ] + in + if List.contains_dup ~compare:String.compare all_nodes_names_list then + failwith + "All nodes in testnet must have unique names. Check to make sure you \ + are not using the same node_name more than once" ; + + (* GENERATE ACCOUNTS AND KEYPAIRS *) + let keypairs = + List.take + (* the first keypair is the genesis winner and is assumed to be untimed. Therefore dropping it, and not assigning it to any block producer *) + (List.drop + (Array.to_list (Lazy.force Key_gen.Sample_keypairs.keypairs)) + 1 ) + (List.length genesis_ledger) + in + let labeled_accounts : + ( Runtime_config.Accounts.single + * (Public_key.Compressed.t * Private_key.t) ) + String.Map.t = + String.Map.empty + in + let rec add_accounts mp zip = + match zip with + | [] -> + mp + | hd :: tl -> + let ( { Test_config.Test_Account.balance; account_name; timing } + , (pk, sk) ) = + hd + in + let timing = + match timing with + | Account.Timing.Untimed -> + None + | Timed t -> + Some + { Runtime_config.Accounts.Single.Timed.initial_minimum_balance = + t.initial_minimum_balance + ; cliff_time = t.cliff_time + ; cliff_amount = t.cliff_amount + ; vesting_period = t.vesting_period + ; vesting_increment = t.vesting_increment + } + in + let default = Runtime_config.Accounts.Single.default in + let acct = + { default with + pk = Public_key.Compressed.to_string pk + ; sk = Some (Private_key.to_base58_check sk) + ; balance = + Balance.of_mina_string_exn balance + (* delegation currently unsupported *) + ; delegate = None + ; timing + } + in + add_accounts + (String.Map.add_exn mp ~key:account_name ~data:(acct, (pk, sk))) + tl + in + let genesis_ledger_accounts = + add_accounts labeled_accounts (List.zip_exn genesis_ledger keypairs) + in + (* DAEMON CONFIG *) + let constraint_constants = + Genesis_ledger_helper.make_constraint_constants + ~default:Genesis_constants.Constraint_constants.compiled proof_config + in + let runtime_config = + { Runtime_config.daemon = + Some + { txpool_max_size = Some txpool_max_size + ; peer_list_url = None + ; zkapp_proof_update_cost = None + ; zkapp_signed_single_update_cost = None + ; zkapp_signed_pair_update_cost = None + ; zkapp_transaction_cost_limit = None + ; max_event_elements = None + ; max_action_elements = None + } + ; genesis = + Some + { k = Some k + ; delta = Some delta + ; slots_per_epoch = Some slots_per_epoch + ; slots_per_sub_window = Some slots_per_sub_window + ; genesis_state_timestamp = + Some Core.Time.(to_string_abs ~zone:Zone.utc (now ())) + } + ; proof = Some proof_config (* TODO: prebake ledger and only set hash *) + ; ledger = + Some + { base = + Accounts + (let tuplist = String.Map.data genesis_ledger_accounts in + List.map tuplist ~f:(fun tup -> + let acct, _ = tup in + acct ) ) + ; add_genesis_winner = None + ; num_accounts = None + ; balances = [] + ; hash = None + ; name = None + } + ; epoch_data = None + } + in + let genesis_constants = + Or_error.ok_exn + (Genesis_ledger_helper.make_genesis_constants ~logger + ~default:Genesis_constants.compiled runtime_config ) + in + let constants : Test_config.constants = + { constraints = constraint_constants; genesis = genesis_constants } + in + (* BLOCK PRODUCER CONFIG *) + let mk_net_keypair keypair_name (pk, sk) = + let keypair = + { Keypair.public_key = Public_key.decompress_exn pk; private_key = sk } + in + Network_keypair.create_network_keypair ~keypair_name ~keypair + in + let block_producer_config name keypair = + { name; keypair; libp2p_secret = "" } + in + let block_producer_configs = + List.map block_producers ~f:(fun node -> + let _, key_tup = + match String.Map.find genesis_ledger_accounts node.account_name with + | Some acct -> + acct + | None -> + let failstring = + Format.sprintf + "Failing because the account key of all initial block \ + producers must be in the genesis ledger. name of Node: \ + %s. name of Account which does not exist: %s" + node.node_name node.account_name + in + failwith failstring + in + block_producer_config node.node_name + (mk_net_keypair node.account_name key_tup) ) + in + let mina_archive_schema = "create_schema.sql" in + let long_commit_id = + if String.is_substring Mina_version.commit_id ~substring:"[DIRTY]" then + String.sub Mina_version.commit_id ~pos:7 + ~len:(String.length Mina_version.commit_id - 7) + else Mina_version.commit_id + in + let mina_archive_base_url = + "https://raw.githubusercontent.com/MinaProtocol/mina/" ^ long_commit_id + ^ "/src/app/archive/" + in + let mina_archive_schema_aux_files = + [ mina_archive_base_url ^ "create_schema.sql" + ; mina_archive_base_url ^ "zkapp_tables.sql" + ] + in + let genesis_keypairs = + String.Map.of_alist_exn + (List.map (String.Map.to_alist genesis_ledger_accounts) + ~f:(fun element -> + let kp_name, (_, (pk, sk)) = element in + (kp_name, mk_net_keypair kp_name (pk, sk)) ) ) + in + let snark_coordinator_config = + match snark_coordinator with + | None -> + None + | Some node -> + let network_kp = + match String.Map.find genesis_keypairs node.account_name with + | Some acct -> + acct + | None -> + let failstring = + Format.sprintf + "Failing because the account key of all initial snark \ + coordinators must be in the genesis ledger. name of \ + Node: %s. name of Account which does not exist: %s" + node.node_name node.account_name + in + failwith failstring + in + Some + { name = node.node_name + ; public_key = + Public_key.Compressed.to_base58_check + (Public_key.compress network_kp.keypair.public_key) + ; worker_nodes = node.worker_nodes + } + in + + (* NETWORK CONFIG *) + { mina_automation_location = cli_inputs.mina_automation_location + ; debug_arg = debug + ; check_capacity = cli_inputs.check_capacity + ; check_capacity_delay = cli_inputs.check_capacity_delay + ; check_capacity_retries = cli_inputs.check_capacity_retries + ; genesis_keypairs + ; constants + ; terraform = + { cluster_name + ; cluster_region + ; k8s_context = cluster_id + ; testnet_name + ; deploy_graphql_ingress = requires_graphql + ; mina_image = images.mina + ; mina_agent_image = images.user_agent + ; mina_bots_image = images.bots + ; mina_points_image = images.points + ; mina_archive_image = images.archive_node + ; runtime_config = Runtime_config.to_yojson runtime_config + ; block_producer_configs + ; log_precomputed_blocks + ; archive_node_count = num_archive_nodes + ; mina_archive_schema + ; mina_archive_schema_aux_files + ; snark_coordinator_config + ; snark_worker_fee + ; aws_route53_zone_id + ; cpu_request = 6 + ; mem_request = "12Gi" + ; worker_cpu_request = 6 + ; worker_mem_request = "8Gi" + } + } + + let to_terraform network_config = + let open Terraform in + [ Block.Terraform + { Block.Terraform.required_version = ">= 0.12.0" + ; backend = + Backend.Local + { path = + "terraform-" ^ network_config.terraform.testnet_name + ^ ".tfstate" + } + } + ; Block.Provider + { Block.Provider.provider = "google" + ; region = cluster_region + ; zone = Some cluster_zone + ; project = Some project_id + ; alias = None + } + ; Block.Module + { Block.Module.local_name = "integration_testnet" + ; providers = [ ("google.gke", "google") ] + ; source = "../../modules/o1-integration" + ; args = terraform_config_to_assoc network_config.terraform + } + ] + + let testnet_log_filter network_config = + Printf.sprintf + {| + resource.labels.project_id="%s" + resource.labels.location="%s" + resource.labels.cluster_name="%s" + resource.labels.namespace_name="%s" + |} + project_id cluster_region cluster_name + network_config.terraform.testnet_name +end + +module Network_id = struct + type t = string [@@deriving eq, yojson] +end + +module Node_id = struct + type t = string [@@deriving eq, yojson] +end + +module Request = struct + type t = + | Access_token + | CreateNetwork of Network_config.t + | DeployNetwork of Network_id.t + | DestroyNetwork of Network_id.t + | GetNodeLogs of Node_id.t + [@@deriving to_yojson] +end + +module Node_type = struct + type t = + | Archive_node + | Block_producer_node + | Non_seed_node + | Seed_node + | Snark_coordinator + [@@deriving eq, yojson] + + let to_string nt = + to_yojson nt |> Yojson.Safe.to_string + + let of_string s = + of_yojson @@ `List [ `String s ] |> Result.ok_or_failwith +end + +module Network_deploy_response = struct + module Map = Map.Make (String) + include Map + + type t = (Node_type.t * string) Map.t [@@deriving eq] + + let of_yojson = + let f accum = function + | node_id, `Tuple [ `String nt; `String gql_uri ] -> + Map.set accum ~key:node_id ~data:(Node_type.of_string nt, gql_uri) + | _ -> + failwith "invalid network_deploy_response yojson entry" + in + function + | `Assoc a_list -> + Ok (List.fold a_list ~init:(Map.empty : t) ~f) + | _ -> + Error "invalid network_deploy_response yojson" +end + +module Access_token = struct + type t = string [@@deriving eq, yojson] +end + +module Response = struct + type t = + | Access_token of Access_token.t + | NetworkCreated of Network_id.t + | NetworkDeployed of Network_deploy_response.t + | NetworkDestroyed + | NodeLogs of Node_id.t * string + [@@deriving eq, of_yojson] +end + +(** + The general workflow could look something like this: + https://www.notion.so/minafoundation/Lucy-CI-Interactions-e36b48ac52994cafbe1367548e02241d?pvs=4 + *) + +let request_ci_access_token () : Response.t Async.Deferred.t = + failwith "request_ci_access_token" + +(* for example, we can communicate with the CI via https and test-specific access token *) +let[@warning "-27"] send_ci_http_request ~(access_token: Access_token.t) ~(request_body: Request.t) : Response.t Async.Deferred.t = + let req_str = request_body |> Request.to_yojson |> Yojson.Safe.to_string in + failwithf "send_ci_http_request: %s\n" req_str () + +module Request_unit_tests = struct + let%test_unit "Create network request" = assert true (* TODO: too complicated for now *) + + let%test_unit "Deploy network request" = + let open Request in + let result = DeployNetwork "network0" |> to_yojson |> Yojson.Safe.to_string in + let ( = ) = String.equal in + assert (result = {| [ "DeployNetwork", "network0" ] |}) + + let%test_unit "Destroy network request" = + let open Request in + let result = DestroyNetwork "network0" |> to_yojson |> Yojson.Safe.to_string in + let ( = ) = String.equal in + assert (result = {| [ "DestroyNetwork", "network0" ] |}) + + let%test_unit "Get node logs request" = + let open Request in + let result = GetNodeLogs "node0" |> to_yojson |> Yojson.Safe.to_string in + let ( = ) = String.equal in + assert (result = {| [ "GetNodeLogs", "node0" ] |}) +end + +module Response_unit_tests = struct + let%test_unit "Parse network created response" = + let open Response in + let result = + `List [ `String "NetworkCreated"; `String "node0" ] + |> of_yojson |> Result.ok_or_failwith + in + let ( = ) = equal in + assert ( + result = NetworkCreated "node0" + ) + + let%test_unit "Parse network deployed response" = + let open Node_type in + let open Network_deploy_response in + let result = + {| { "node0": ("Archive_node", "gql0"), + "node1": ("Block_producer_node", "gql1"), + "node2": ("Non_seed_node", "gql2"), + "node3": ("Seed_node", "gql3"), + "node4": ("Snark_coordinator", "gql4") + } |} + |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith + in + let ( = ) = equal in + assert ( + result + = of_alist_exn + [ ("node0", (Archive_node, "gql0")) + ; ("node1", (Block_producer_node, "gql1")) + ; ("node2", (Non_seed_node, "gql2")) + ; ("node3", (Seed_node, "gql3")) + ; ("node4", (Snark_coordinator, "gql4")) + ] ) + + let%test_unit "Parse network destroyed response" = + let open Response in + let result = + `List [ `String "NetworkDestroyed" ] + |> of_yojson |> Result.ok_or_failwith + in + assert ( + equal result NetworkDestroyed + ) + + let%test_unit "Parse node logs response" = + let open Response in + let result = + `List [ `String "NodeLogs"; `String "node0"; `String "node0_logs" ] + |> of_yojson |> Result.ok_or_failwith + in + let ( = ) = equal in + assert ( + result = NodeLogs ("node0", "node0_logs") + ) +end \ No newline at end of file From 8a2e0d0ad5f313f7e464c9e7778a43c1bdbfb97d Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 25 Jul 2023 04:21:36 +0000 Subject: [PATCH 04/90] Add abstract network with some ci interactions --- .../abstract_network.ml | 1456 +++++++++++++++++ 1 file changed, 1456 insertions(+) create mode 100644 src/lib/integration_test_abstract_engine/abstract_network.ml diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml new file mode 100644 index 00000000000..51f1bae74a6 --- /dev/null +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -0,0 +1,1456 @@ +open Core_kernel +open Async +open Integration_test_lib +open Mina_transaction +open Ci_interaction + +(* exclude from bisect_ppx to avoid type error on GraphQL modules *) +[@@@coverage exclude_file] + +let mina_archive_container_id = "mina_archive_container_id" + +let mina_archive_username = "mina_archive_username" + +let mina_archive_pw = "mina_archive_pw" + +let postgres_url = + Printf.sprintf "postgres://%s:%s@archive-1-postgresql:5432/archive" + mina_archive_username mina_archive_pw + +let node_password = "naughty blue worm" + +type config = + { testnet_name : string + ; cluster : string + ; namespace : string + ; graphql_enabled : bool + ; access_token : Access_token.t + } + +let base_kube_args { cluster; namespace; _ } = + [ "--cluster"; cluster; "--namespace"; namespace ] + +let id : Network_id.t Ivar.t = Async.Ivar.create () + +let network_ingress_uri : string Ivar.t = Async.Ivar.create () + +module Node = struct + type pod_info = + { network_keypair : Network_keypair.t option + ; primary_container_id : string + (* this is going to be probably either "mina" or "worker" *) + ; has_archive_container : bool + (* archive pods have a "mina" container and an "archive" container alongside *) + } + + type t = + { app_id : string + ; pod_ids : string list + ; pod_info : pod_info + ; config : config + } + + let id { pod_ids; _ } = List.hd_exn pod_ids + + let network_keypair { pod_info = { network_keypair; _ }; _ } = network_keypair + + let base_kube_args t = [ "--cluster"; t.cluster; "--namespace"; t.namespace ] + + let get_logs_in_container ?container_id { pod_ids; config; pod_info; _ } = + let container_id = + Option.value container_id ~default:pod_info.primary_container_id + in + let%bind cwd = Unix.getcwd () in + Integration_test_lib.Util.run_cmd_or_hard_error ~exit_code:13 cwd "kubectl" + ( base_kube_args config + @ [ "logs"; "-c"; container_id; List.hd_exn pod_ids ] ) + + let run_in_container ?(exit_code = 10) ?container_id ?override_with_pod_id + ~cmd t = + let { config; pod_info; _ } = t in + let pod_id = + match override_with_pod_id with + | Some pid -> + pid + | None -> + List.hd_exn t.pod_ids + in + let container_id = + Option.value container_id ~default:pod_info.primary_container_id + in + let%bind cwd = Unix.getcwd () in + Integration_test_lib.Util.run_cmd_or_hard_error ~exit_code cwd "kubectl" + ( base_kube_args config + @ [ "exec"; "-c"; container_id; "-i"; pod_id; "--" ] + @ cmd ) + + let cp_string_to_container_file ?container_id ~str ~dest t = + let { pod_ids; config; pod_info; _ } = t in + let container_id = + Option.value container_id ~default:pod_info.primary_container_id + in + let tmp_file, oc = + Caml.Filename.open_temp_file ~temp_dir:Filename.temp_dir_name + "integration_test_cp_string" ".tmp" + in + Out_channel.output_string oc str ; + Out_channel.close oc ; + let%bind cwd = Unix.getcwd () in + let dest_file = + sprintf "%s/%s:%s" config.namespace (List.hd_exn pod_ids) dest + in + Integration_test_lib.Util.run_cmd_or_error cwd "kubectl" + (base_kube_args config @ [ "cp"; "-c"; container_id; tmp_file; dest_file ]) + + let start ~fresh_state node : unit Malleable_error.t = + let open Malleable_error.Let_syntax in + let%bind () = + if fresh_state then + run_in_container node ~cmd:[ "sh"; "-c"; "rm -rf .mina-config/*" ] + >>| ignore + else Malleable_error.return () + in + run_in_container ~exit_code:11 node ~cmd:[ "/start.sh" ] >>| ignore + + let stop node = + let open Malleable_error.Let_syntax in + run_in_container ~exit_code:12 node ~cmd:[ "/stop.sh" ] >>| ignore + + let logger_metadata node = + [ ("namespace", `String node.config.namespace) + ; ("app_id", `String node.app_id) + ; ("pod_id", `String (List.hd_exn node.pod_ids)) + ] + + module Scalars = Graphql_lib.Scalars + + module Graphql = struct + let ingress_uri node = + let host = + sprintf "%s.%s" node.config.testnet_name + (Ivar.value_exn network_ingress_uri) + in + let path = sprintf "/%s/graphql" node.app_id in + Uri.make ~scheme:"http" ~host ~path ~port:80 () + + module Client = Graphql_lib.Client.Make (struct + let preprocess_variables_string = Fn.id + + let headers = String.Map.empty + end) + + (* graphql_ppx uses Stdlib symbols instead of Base *) + open Stdlib + module Encoders = Mina_graphql.Types.Input + + module Unlock_account = + [%graphql + ({| + mutation ($password: String!, $public_key: PublicKey!) @encoders(module: "Encoders"){ + unlockAccount(input: {password: $password, publicKey: $public_key }) { + account { + public_key: publicKey + } + } + } + |} + [@encoders Encoders] )] + + module Send_test_payments = + [%graphql + {| + mutation ($senders: [PrivateKey!]!, + $receiver: PublicKey!, + $amount: UInt64!, + $fee: UInt64!, + $repeat_count: UInt32!, + $repeat_delay_ms: UInt32!) @encoders(module: "Encoders"){ + sendTestPayments( + senders: $senders, receiver: $receiver, amount: $amount, fee: $fee, + repeat_count: $repeat_count, + repeat_delay_ms: $repeat_delay_ms) + } + |}] + + module Send_payment = + [%graphql + {| + mutation ($input: SendPaymentInput!)@encoders(module: "Encoders"){ + sendPayment(input: $input){ + payment { + id + nonce + hash + } + } + } + |}] + + module Send_payment_with_raw_sig = + [%graphql + {| + mutation ( + $input:SendPaymentInput!, + $rawSignature: String! + )@encoders(module: "Encoders") + { + sendPayment( + input:$input, + signature: {rawSignature: $rawSignature} + ) + { + payment { + id + nonce + hash + } + } + } + |}] + + module Send_delegation = + [%graphql + {| + mutation ($input: SendDelegationInput!) @encoders(module: "Encoders"){ + sendDelegation(input:$input){ + delegation { + id + nonce + hash + } + } + } + |}] + + module Set_snark_worker = + [%graphql + {| + mutation ($input: SetSnarkWorkerInput! ) @encoders(module: "Encoders"){ + setSnarkWorker(input:$input){ + lastSnarkWorker + } + } + |}] + + module Get_account_data = + [%graphql + {| + query ($public_key: PublicKey!) @encoders(module: "Encoders"){ + account(publicKey: $public_key) { + nonce + balance { + total @ppxCustom(module: "Scalars.Balance") + } + } + } + |}] + + module Send_test_zkapp = Generated_graphql_queries.Send_test_zkapp + module Pooled_zkapp_commands = + Generated_graphql_queries.Pooled_zkapp_commands + + module Query_peer_id = + [%graphql + {| + query { + daemonStatus { + addrsAndPorts { + peer { + peerId + } + } + peers { peerId } + + } + } + |}] + + module Best_chain = + [%graphql + {| + query ($max_length: Int) @encoders(module: "Encoders"){ + bestChain (maxLength: $max_length) { + stateHash @ppxCustom(module: "Graphql_lib.Scalars.String_json") + commandTransactionCount + creatorAccount { + publicKey @ppxCustom(module: "Graphql_lib.Scalars.JSON") + } + } + } + |}] + + module Query_metrics = + [%graphql + {| + query { + daemonStatus { + metrics { + blockProductionDelay + transactionPoolDiffReceived + transactionPoolDiffBroadcasted + transactionsAddedToPool + transactionPoolSize + } + } + } + |}] + + module StartFilteredLog = + [%graphql + {| + mutation ($filter: [String!]!) @encoders(module: "Encoders"){ + startFilteredLog(filter: $filter) + } + |}] + + module GetFilteredLogEntries = + [%graphql + {| + query ($offset: Int!) @encoders(module: "Encoders"){ + getFilteredLogEntries(offset: $offset) { + logMessages, + isCapturing, + } + + } + |}] + + module Account = + [%graphql + {| + query ($public_key: PublicKey!, $token: UInt64) { + account (publicKey : $public_key, token : $token) { + balance { liquid + locked + total + } + delegate + nonce + permissions { editActionState + editState + incrementNonce + receive + send + access + setDelegate + setPermissions + setZkappUri + setTokenSymbol + setVerificationKey + setVotingFor + setTiming + } + actionState + zkappState + zkappUri + timing { cliffTime @ppxCustom(module: "Graphql_lib.Scalars.JSON") + cliffAmount + vestingPeriod @ppxCustom(module: "Graphql_lib.Scalars.JSON") + vestingIncrement + initialMinimumBalance + } + tokenId + tokenSymbol + verificationKey { verificationKey + hash + } + votingFor + } + } + |}] + end + + (* this function will repeatedly attempt to connect to graphql port times before giving up *) + let exec_graphql_request ?(num_tries = 10) ?(retry_delay_sec = 30.0) + ?(initial_delay_sec = 0.) ~logger ~node ~query_name query_obj = + let open Deferred.Let_syntax in + if not node.config.graphql_enabled then + Deferred.Or_error.error_string + "graphql is not enabled (hint: set `requires_graphql = true` in the \ + test config)" + else + let uri = Graphql.ingress_uri node in + let metadata = + [ ("query", `String query_name) + ; ("uri", `String (Uri.to_string uri)) + ; ("init_delay", `Float initial_delay_sec) + ] + in + [%log info] + "Attempting to send GraphQL request \"$query\" to \"$uri\" after \ + $init_delay sec" + ~metadata ; + let rec retry n = + if n <= 0 then ( + [%log error] + "GraphQL request \"$query\" to \"$uri\" failed too many times" + ~metadata ; + Deferred.Or_error.errorf + "GraphQL \"%s\" to \"%s\" request failed too many times" query_name + (Uri.to_string uri) ) + else + match%bind Graphql.Client.query query_obj uri with + | Ok result -> + [%log info] "GraphQL request \"$query\" to \"$uri\" succeeded" + ~metadata ; + Deferred.Or_error.return result + | Error (`Failed_request err_string) -> + [%log warn] + "GraphQL request \"$query\" to \"$uri\" failed: \"$error\" \ + ($num_tries attempts left)" + ~metadata: + ( metadata + @ [ ("error", `String err_string) + ; ("num_tries", `Int (n - 1)) + ] ) ; + let%bind () = after (Time.Span.of_sec retry_delay_sec) in + retry (n - 1) + | Error (`Graphql_error err_string) -> + [%log error] + "GraphQL request \"$query\" to \"$uri\" returned an error: \ + \"$error\" (this is a graphql error so not retrying)" + ~metadata:(metadata @ [ ("error", `String err_string) ]) ; + Deferred.Or_error.error_string err_string + in + let%bind () = after (Time.Span.of_sec initial_delay_sec) in + retry num_tries + + let get_peer_ids ~logger t = + let open Deferred.Or_error.Let_syntax in + [%log info] "Getting node's peer_id, and the peer_ids of node's peers" + ~metadata:(logger_metadata t) ; + let query_obj = Graphql.Query_peer_id.(make @@ makeVariables ()) in + let%bind query_result_obj = + exec_graphql_request ~logger ~node:t ~query_name:"query_peer_id" query_obj + in + [%log info] "get_peer_ids, finished exec_graphql_request" ; + let self_id_obj = query_result_obj.daemonStatus.addrsAndPorts.peer in + let%bind self_id = + match self_id_obj with + | None -> + Deferred.Or_error.error_string "Peer not found" + | Some peer -> + return peer.peerId + in + let peers = query_result_obj.daemonStatus.peers |> Array.to_list in + let peer_ids = List.map peers ~f:(fun peer -> peer.peerId) in + [%log info] + "get_peer_ids, result of graphql query (self_id,[peers]) (%s,%s)" self_id + (String.concat ~sep:" " peer_ids) ; + return (self_id, peer_ids) + + let must_get_peer_ids ~logger t = + get_peer_ids ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error + + let get_best_chain ?max_length ~logger t = + let open Deferred.Or_error.Let_syntax in + let query = Graphql.Best_chain.(make @@ makeVariables ?max_length ()) in + let%bind result = + exec_graphql_request ~logger ~node:t ~query_name:"best_chain" query + in + match result.bestChain with + | None | Some [||] -> + Deferred.Or_error.error_string "failed to get best chains" + | Some chain -> + return + @@ List.map + ~f:(fun block -> + Intf. + { state_hash = block.stateHash + ; command_transaction_count = block.commandTransactionCount + ; creator_pk = + ( match block.creatorAccount.publicKey with + | `String pk -> + pk + | _ -> + "unknown" ) + } ) + (Array.to_list chain) + + let must_get_best_chain ?max_length ~logger t = + get_best_chain ?max_length ~logger t + |> Deferred.bind ~f:Malleable_error.or_hard_error + + let get_account ~logger t ~account_id = + let pk = Mina_base.Account_id.public_key account_id in + let token = Mina_base.Account_id.token_id account_id in + [%log info] "Getting account" + ~metadata: + ( ("pub_key", Signature_lib.Public_key.Compressed.to_yojson pk) + :: logger_metadata t ) ; + let get_account_obj = + Graphql.Account.( + make + @@ makeVariables + ~public_key:(Graphql_lib.Encoders.public_key pk) + ~token:(Graphql_lib.Encoders.token token) + ()) + in + exec_graphql_request ~logger ~node:t ~query_name:"get_account_graphql" + get_account_obj + + type account_data = + { nonce : Mina_numbers.Account_nonce.t + ; total_balance : Currency.Balance.t + ; liquid_balance_opt : Currency.Balance.t option + ; locked_balance_opt : Currency.Balance.t option + } + + let get_account_data ~logger t ~account_id = + let open Deferred.Or_error.Let_syntax in + let public_key = Mina_base.Account_id.public_key account_id in + let token = Mina_base.Account_id.token_id account_id in + [%log info] "Getting account data, which is its balances and nonce" + ~metadata: + ( ("pub_key", Signature_lib.Public_key.Compressed.to_yojson public_key) + :: logger_metadata t ) ; + let%bind account_obj = get_account ~logger t ~account_id in + match account_obj.account with + | None -> + Deferred.Or_error.errorf + !"Account with Account id %{sexp:Mina_base.Account_id.t}, public_key \ + %s, and token %s not found" + account_id + (Signature_lib.Public_key.Compressed.to_string public_key) + (Mina_base.Token_id.to_string token) + | Some acc -> + return + { nonce = + Option.value_exn + ~message: + "the nonce from get_balance is None, which should be \ + impossible" + acc.nonce + ; total_balance = acc.balance.total + ; liquid_balance_opt = acc.balance.liquid + ; locked_balance_opt = acc.balance.locked + } + + let must_get_account_data ~logger t ~account_id = + get_account_data ~logger t ~account_id + |> Deferred.bind ~f:Malleable_error.or_hard_error + + let permissions_of_account_permissions account_permissions : + Mina_base.Permissions.t = + (* the polymorphic variants come from Partial_accounts.auth_required in Mina_graphql *) + let to_auth_required = function + | `Either -> + Mina_base.Permissions.Auth_required.Either + | `Impossible -> + Impossible + | `None -> + None + | `Proof -> + Proof + | `Signature -> + Signature + in + let open Graphql.Account in + { edit_action_state = to_auth_required account_permissions.editActionState + ; edit_state = to_auth_required account_permissions.editState + ; increment_nonce = to_auth_required account_permissions.incrementNonce + ; receive = to_auth_required account_permissions.receive + ; send = to_auth_required account_permissions.send + ; access = to_auth_required account_permissions.access + ; set_delegate = to_auth_required account_permissions.setDelegate + ; set_permissions = to_auth_required account_permissions.setPermissions + ; set_zkapp_uri = to_auth_required account_permissions.setZkappUri + ; set_token_symbol = to_auth_required account_permissions.setTokenSymbol + ; set_verification_key = + to_auth_required account_permissions.setVerificationKey + ; set_voting_for = to_auth_required account_permissions.setVotingFor + ; set_timing = to_auth_required account_permissions.setTiming + } + + let graphql_uri node = Graphql.ingress_uri node |> Uri.to_string + + let get_account_permissions ~logger t ~account_id = + let open Deferred.Or_error in + let open Let_syntax in + let%bind account_obj = get_account ~logger t ~account_id in + match account_obj.account with + | Some account -> ( + match account.permissions with + | Some ledger_permissions -> + return @@ permissions_of_account_permissions ledger_permissions + | None -> + fail + (Error.of_string "Could not get permissions from ledger account") + ) + | None -> + fail (Error.of_string "Could not get account from ledger") + + (* return an Account_update.Update.t with all fields `Set` to the + value in the account, or `Keep` if value unavailable, + as if this update had been applied to the account + *) + let get_account_update ~logger t ~account_id = + let open Deferred.Or_error in + let open Let_syntax in + let%bind account_obj = get_account ~logger t ~account_id in + match account_obj.account with + | Some account -> + let open Mina_base.Zkapp_basic.Set_or_keep in + let%bind app_state = + match account.zkappState with + | Some strs -> + let fields = + Array.to_list strs |> Base.List.map ~f:(fun s -> Set s) + in + return (Mina_base.Zkapp_state.V.of_list_exn fields) + | None -> + fail + (Error.of_string + (sprintf + "Expected zkApp account with an app state for public key \ + %s" + (Signature_lib.Public_key.Compressed.to_base58_check + (Mina_base.Account_id.public_key account_id) ) ) ) + in + let%bind delegate = + match account.delegate with + | Some s -> + return (Set s) + | None -> + fail (Error.of_string "Expected delegate in account") + in + let%bind verification_key = + match account.verificationKey with + | Some vk_obj -> + let data = vk_obj.verificationKey in + let hash = vk_obj.hash in + return (Set ({ data; hash } : _ With_hash.t)) + | None -> + fail + (Error.of_string + (sprintf + "Expected zkApp account with a verification key for \ + public_key %s" + (Signature_lib.Public_key.Compressed.to_base58_check + (Mina_base.Account_id.public_key account_id) ) ) ) + in + let%bind permissions = + match account.permissions with + | Some perms -> + return @@ Set (permissions_of_account_permissions perms) + | None -> + fail (Error.of_string "Expected permissions in account") + in + let%bind zkapp_uri = + match account.zkappUri with + | Some s -> + return @@ Set s + | None -> + fail (Error.of_string "Expected zkApp URI in account") + in + let%bind token_symbol = + match account.tokenSymbol with + | Some s -> + return @@ Set s + | None -> + fail (Error.of_string "Expected token symbol in account") + in + let%bind timing = + let timing = account.timing in + let cliff_amount = timing.cliffAmount in + let cliff_time = timing.cliffTime in + let vesting_period = timing.vestingPeriod in + let vesting_increment = timing.vestingIncrement in + let initial_minimum_balance = timing.initialMinimumBalance in + match + ( cliff_amount + , cliff_time + , vesting_period + , vesting_increment + , initial_minimum_balance ) + with + | None, None, None, None, None -> + return @@ Keep + | Some amt, Some tm, Some period, Some incr, Some bal -> + let cliff_amount = amt in + let%bind cliff_time = + match tm with + | `String s -> + return @@ Mina_numbers.Global_slot_since_genesis.of_string s + | _ -> + fail + (Error.of_string + "Expected string for cliff time in account timing" ) + in + let%bind vesting_period = + match period with + | `String s -> + return @@ Mina_numbers.Global_slot_span.of_string s + | _ -> + fail + (Error.of_string + "Expected string for vesting period in account timing" ) + in + let vesting_increment = incr in + let initial_minimum_balance = bal in + return + (Set + ( { initial_minimum_balance + ; cliff_amount + ; cliff_time + ; vesting_period + ; vesting_increment + } + : Mina_base.Account_update.Update.Timing_info.t ) ) + | _ -> + fail (Error.of_string "Some pieces of account timing are missing") + in + let%bind voting_for = + match account.votingFor with + | Some s -> + return @@ Set s + | None -> + fail (Error.of_string "Expected voting-for state hash in account") + in + return + ( { app_state + ; delegate + ; verification_key + ; permissions + ; zkapp_uri + ; token_symbol + ; timing + ; voting_for + } + : Mina_base.Account_update.Update.t ) + | None -> + fail (Error.of_string "Could not get account from ledger") + + type signed_command_result = + { id : string + ; hash : Transaction_hash.t + ; nonce : Mina_numbers.Account_nonce.t + } + + let transaction_id_to_string id = + Yojson.Basic.to_string (Graphql_lib.Scalars.TransactionId.serialize id) + + (* if we expect failure, might want retry_on_graphql_error to be false *) + let send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee = + [%log info] "Sending a payment" ~metadata:(logger_metadata t) ; + let open Deferred.Or_error.Let_syntax in + let sender_pk_str = + Signature_lib.Public_key.Compressed.to_string sender_pub_key + in + [%log info] "send_payment: unlocking account" + ~metadata:[ ("sender_pk", `String sender_pk_str) ] ; + let unlock_sender_account_graphql () = + let unlock_account_obj = + Graphql.Unlock_account.( + make + @@ makeVariables ~password:node_password ~public_key:sender_pub_key ()) + in + exec_graphql_request ~logger ~node:t ~initial_delay_sec:0. + ~query_name:"unlock_sender_account_graphql" unlock_account_obj + in + let%bind _unlock_acct_obj = unlock_sender_account_graphql () in + let send_payment_graphql () = + let input = + Mina_graphql.Types.Input.SendPaymentInput.make_input + ~from:sender_pub_key ~to_:receiver_pub_key ~amount ~fee () + in + let send_payment_obj = + Graphql.Send_payment.(make @@ makeVariables ~input ()) + in + exec_graphql_request ~logger ~node:t ~query_name:"send_payment_graphql" + send_payment_obj + in + let%map sent_payment_obj = send_payment_graphql () in + let return_obj = sent_payment_obj.sendPayment.payment in + let res = + { id = transaction_id_to_string return_obj.id + ; hash = return_obj.hash + ; nonce = Mina_numbers.Account_nonce.of_int return_obj.nonce + } + in + [%log info] "Sent payment" + ~metadata: + [ ("user_command_id", `String res.id) + ; ("hash", `String (Transaction_hash.to_base58_check res.hash)) + ; ("nonce", `Int (Mina_numbers.Account_nonce.to_int res.nonce)) + ] ; + res + + let must_send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee + = + send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee + |> Deferred.bind ~f:Malleable_error.or_hard_error + + let send_zkapp_batch ~logger (t : t) + ~(zkapp_commands : Mina_base.Zkapp_command.t list) = + [%log info] "Sending zkapp transactions" + ~metadata: + [ ("namespace", `String t.config.namespace) + ; ("pod_id", `String (id t)) + ] ; + let open Deferred.Or_error.Let_syntax in + let zkapp_commands_json = + List.map zkapp_commands ~f:(fun zkapp_command -> + Mina_base.Zkapp_command.to_json zkapp_command |> Yojson.Safe.to_basic ) + |> Array.of_list + in + let send_zkapp_graphql () = + let send_zkapp_obj = + Graphql.Send_test_zkapp.( + make @@ makeVariables ~zkapp_commands:zkapp_commands_json ()) + in + exec_graphql_request ~logger ~node:t ~query_name:"send_zkapp_graphql" + send_zkapp_obj + in + let%bind sent_zkapp_obj = send_zkapp_graphql () in + let%bind zkapp_ids = + Deferred.Array.fold ~init:(Ok []) sent_zkapp_obj.internalSendZkapp + ~f:(fun acc (zkapp_obj : Graphql.Send_test_zkapp.t_internalSendZkapp) -> + let%bind res = + match zkapp_obj.zkapp.failureReason with + | None -> + let zkapp_id = transaction_id_to_string zkapp_obj.zkapp.id in + [%log info] "Sent zkapp transaction" + ~metadata:[ ("zkapp_id", `String zkapp_id) ] ; + return zkapp_id + | Some s -> + Deferred.Or_error.errorf "Zkapp failed, reason: %s" + ( Array.fold ~init:[] s ~f:(fun acc f -> + match f with + | None -> + acc + | Some f -> + let t = + ( Option.value_exn f.index + , f.failures |> Array.to_list |> List.rev ) + in + t :: acc ) + |> Mina_base.Transaction_status.Failure.Collection.Display + .to_yojson |> Yojson.Safe.to_string ) + in + let%map acc = Deferred.return acc in + res :: acc ) + in + return (List.rev zkapp_ids) + + let get_pooled_zkapp_commands ~logger (t : t) + ~(pk : Signature_lib.Public_key.Compressed.t) = + [%log info] "Retrieving zkapp_commands from transaction pool" + ~metadata: + [ ("namespace", `String t.config.namespace) + ; ("pod_id", `String (id t)) + ; ("pub_key", Signature_lib.Public_key.Compressed.to_yojson pk) + ] ; + let open Deferred.Or_error.Let_syntax in + let get_pooled_zkapp_commands_graphql () = + let get_pooled_zkapp_commands = + Graphql.Pooled_zkapp_commands.( + make + @@ makeVariables ~public_key:(Graphql_lib.Encoders.public_key pk) ()) + in + exec_graphql_request ~logger ~node:t + ~query_name:"get_pooled_zkapp_commands" get_pooled_zkapp_commands + in + let%bind zkapp_pool_obj = get_pooled_zkapp_commands_graphql () in + let transaction_ids = + Array.map zkapp_pool_obj.pooledZkappCommands ~f:(fun zkapp_command -> + zkapp_command.id |> Transaction_id.to_base64 ) + |> Array.to_list + in + [%log info] "Retrieved zkapp_commands from transaction pool" + ~metadata: + [ ("namespace", `String t.config.namespace) + ; ("pod_id", `String (id t)) + ; ( "transaction ids" + , `List (List.map ~f:(fun t -> `String t) transaction_ids) ) + ] ; + return transaction_ids + + let send_delegation ~logger t ~sender_pub_key ~receiver_pub_key ~fee = + [%log info] "Sending stake delegation" ~metadata:(logger_metadata t) ; + let open Deferred.Or_error.Let_syntax in + let sender_pk_str = + Signature_lib.Public_key.Compressed.to_string sender_pub_key + in + [%log info] "send_delegation: unlocking account" + ~metadata:[ ("sender_pk", `String sender_pk_str) ] ; + let unlock_sender_account_graphql () = + let unlock_account_obj = + Graphql.Unlock_account.( + make + @@ makeVariables ~password:"naughty blue worm" + ~public_key:sender_pub_key ()) + in + exec_graphql_request ~logger ~node:t + ~query_name:"unlock_sender_account_graphql" unlock_account_obj + in + let%bind _ = unlock_sender_account_graphql () in + let send_delegation_graphql () = + let input = + Mina_graphql.Types.Input.SendDelegationInput.make_input + ~from:sender_pub_key ~to_:receiver_pub_key ~fee () + in + let send_delegation_obj = + Graphql.Send_delegation.(make @@ makeVariables ~input ()) + in + exec_graphql_request ~logger ~node:t ~query_name:"send_delegation_graphql" + send_delegation_obj + in + let%map result_obj = send_delegation_graphql () in + let return_obj = result_obj.sendDelegation.delegation in + let res = + { id = transaction_id_to_string return_obj.id + ; hash = return_obj.hash + ; nonce = Mina_numbers.Account_nonce.of_int return_obj.nonce + } + in + [%log info] "stake delegation sent" + ~metadata: + [ ("user_command_id", `String res.id) + ; ("hash", `String (Transaction_hash.to_base58_check res.hash)) + ; ("nonce", `Int (Mina_numbers.Account_nonce.to_int res.nonce)) + ] ; + res + + let must_send_delegation ~logger t ~sender_pub_key ~receiver_pub_key ~fee = + send_delegation ~logger t ~sender_pub_key ~receiver_pub_key ~fee + |> Deferred.bind ~f:Malleable_error.or_hard_error + + let send_payment_with_raw_sig ~logger t ~sender_pub_key ~receiver_pub_key + ~amount ~fee ~nonce ~memo + ~(valid_until : Mina_numbers.Global_slot_since_genesis.t) ~raw_signature = + [%log info] "Sending a payment with raw signature" + ~metadata:(logger_metadata t) ; + let open Deferred.Or_error.Let_syntax in + let send_payment_graphql () = + let open Graphql.Send_payment_with_raw_sig in + let input = + Mina_graphql.Types.Input.SendPaymentInput.make_input + ~from:sender_pub_key ~to_:receiver_pub_key ~amount ~fee ~memo ~nonce + ~valid_until: + (Mina_numbers.Global_slot_since_genesis.to_uint32 valid_until) + () + in + let variables = makeVariables ~input ~rawSignature:raw_signature () in + let send_payment_obj = make variables in + let variables_json_basic = + variablesToJson (serializeVariables variables) + in + (* An awkward conversion from Yojson.Basic to Yojson.Safe *) + let variables_json = + Yojson.Basic.to_string variables_json_basic |> Yojson.Safe.from_string + in + [%log info] "send_payment_obj with $variables " + ~metadata:[ ("variables", variables_json) ] ; + exec_graphql_request ~logger ~node:t + ~query_name:"Send_payment_with_raw_sig_graphql" send_payment_obj + in + let%map sent_payment_obj = send_payment_graphql () in + let return_obj = sent_payment_obj.sendPayment.payment in + let res = + { id = transaction_id_to_string return_obj.id + ; hash = return_obj.hash + ; nonce = Mina_numbers.Account_nonce.of_int return_obj.nonce + } + in + [%log info] "Sent payment" + ~metadata: + [ ("user_command_id", `String res.id) + ; ("hash", `String (Transaction_hash.to_base58_check res.hash)) + ; ("nonce", `Int (Mina_numbers.Account_nonce.to_int res.nonce)) + ] ; + res + + let must_send_payment_with_raw_sig ~logger t ~sender_pub_key ~receiver_pub_key + ~amount ~fee ~nonce ~memo ~valid_until ~raw_signature = + send_payment_with_raw_sig ~logger t ~sender_pub_key ~receiver_pub_key + ~amount ~fee ~nonce ~memo ~valid_until ~raw_signature + |> Deferred.bind ~f:Malleable_error.or_hard_error + + let send_test_payments ~repeat_count ~repeat_delay_ms ~logger t ~senders + ~receiver_pub_key ~amount ~fee = + [%log info] "Sending a series of test payments" + ~metadata:(logger_metadata t) ; + let open Deferred.Or_error.Let_syntax in + let send_payment_graphql () = + let send_payment_obj = + Graphql.Send_test_payments.( + make + @@ makeVariables ~senders ~receiver:receiver_pub_key + ~amount:(Currency.Amount.to_uint64 amount) + ~fee:(Currency.Fee.to_uint64 fee) + ~repeat_count ~repeat_delay_ms ()) + in + exec_graphql_request ~logger ~node:t ~query_name:"send_payment_graphql" + send_payment_obj + in + let%map _ = send_payment_graphql () in + [%log info] "Sent test payments" + + let must_send_test_payments ~repeat_count ~repeat_delay_ms ~logger t ~senders + ~receiver_pub_key ~amount ~fee = + send_test_payments ~repeat_count ~repeat_delay_ms ~logger t ~senders + ~receiver_pub_key ~amount ~fee + |> Deferred.bind ~f:Malleable_error.or_hard_error + + let set_snark_worker ~logger t ~new_snark_pub_key = + [%log info] "Changing snark worker key" ~metadata:(logger_metadata t) ; + let open Deferred.Or_error.Let_syntax in + let set_snark_worker_graphql () = + let input = Some new_snark_pub_key in + let set_snark_worker_obj = + Graphql.Set_snark_worker.(make @@ makeVariables ~input ()) + in + exec_graphql_request ~logger ~node:t + ~query_name:"set_snark_worker_graphql" set_snark_worker_obj + in + let%map result_obj = set_snark_worker_graphql () in + let returned_last_snark_worker_opt = + result_obj.setSnarkWorker.lastSnarkWorker + in + let last_snark_worker = + match returned_last_snark_worker_opt with + | None -> + "" + | Some last -> + last |> Scalars.PublicKey.serialize |> Yojson.Basic.to_string + in + [%log info] "snark worker changed, lastSnarkWorker: %s" last_snark_worker + ~metadata:[ ("lastSnarkWorker", `String last_snark_worker) ] ; + () + + let must_set_snark_worker ~logger t ~new_snark_pub_key = + set_snark_worker ~logger t ~new_snark_pub_key + |> Deferred.bind ~f:Malleable_error.or_hard_error + + let start_filtered_log ~logger ~log_filter t = + let open Deferred.Let_syntax in + let query_obj = + Graphql.StartFilteredLog.(make @@ makeVariables ~filter:log_filter ()) + in + let%bind res = + exec_graphql_request ~logger:(Logger.null ()) ~retry_delay_sec:10.0 + ~node:t ~query_name:"StartFilteredLog" query_obj + in + match res with + | Ok query_result_obj -> + let had_already_started = query_result_obj.startFilteredLog in + if had_already_started then return (Ok ()) + else ( + [%log error] + "Attempted to start structured log collection on $node, but it had \ + already started" + ~metadata:[ ("node", `String t.app_id) ] ; + return (Ok ()) ) + | Error e -> + return (Error e) + + let get_filtered_log_entries ~last_log_index_seen t = + let open Deferred.Or_error.Let_syntax in + let query_obj = + Graphql.GetFilteredLogEntries.( + make @@ makeVariables ~offset:last_log_index_seen ()) + in + let%bind query_result_obj = + exec_graphql_request ~logger:(Logger.null ()) ~retry_delay_sec:10.0 + ~node:t ~query_name:"GetFilteredLogEntries" query_obj + in + let res = query_result_obj.getFilteredLogEntries in + if res.isCapturing then return res.logMessages + else + Deferred.Or_error.error_string + "Node is not currently capturing structured log messages" + + (* return an error if [t] isn't an archive node *) + let dump_archive_data ~logger (t : t) ~data_file = + let%bind data = + match%map send_ci_http_request ~access_token:t.config.access_token ~request_body:(Request.GetNodeLogs (id t)) with + | Response.NodeLogs (_, logs) -> logs + | _ -> failwith "invalid node logs response" + in + [%log info] "Dumping archive data to file %s" data_file ; + Malleable_error.return @@ + Out_channel.with_file data_file ~f:(fun out_ch -> + Out_channel.output_string out_ch data ) + + let run_replayer ~logger (t : t) = + [%log info] "Running replayer on archived data (node: %s, container: %s)" + (List.hd_exn t.pod_ids) mina_archive_container_id ; + let open Malleable_error.Let_syntax in + let%bind accounts = + run_in_container t + ~cmd:[ "jq"; "-c"; ".ledger.accounts"; "/root/config/daemon.json" ] + in + let replayer_input = + sprintf + {| { "genesis_ledger": { "accounts": %s, "add_genesis_winner": true }} |} + accounts + in + let dest = "replayer-input.json" in + let%bind _res = + Deferred.bind ~f:Malleable_error.return + (cp_string_to_container_file t ~container_id:mina_archive_container_id + ~str:replayer_input ~dest ) + in + run_in_container t ~container_id:mina_archive_container_id + ~cmd: + [ "mina-replayer" + ; "--archive-uri" + ; postgres_url + ; "--input-file" + ; dest + ; "--output-file" + ; "/dev/null" + ; "--continue-on-error" + ] + + let dump_mina_logs ~logger (t : t) ~log_file = + let open Malleable_error.Let_syntax in + [%log info] "Dumping container logs from (node: %s, container: %s)" + (List.hd_exn t.pod_ids) t.pod_info.primary_container_id ; + let%map logs = get_logs_in_container t in + [%log info] "Dumping container log to file %s" log_file ; + Out_channel.with_file log_file ~f:(fun out_ch -> + Out_channel.output_string out_ch logs ) + + let dump_precomputed_blocks ~logger (t : t) = + let open Malleable_error.Let_syntax in + [%log info] + "Dumping precomputed blocks from logs for (node: %s, container: %s)" + (List.hd_exn t.pod_ids) t.pod_info.primary_container_id ; + let%bind logs = get_logs_in_container t in + (* kubectl logs may include non-log output, like "Using password from environment variable" *) + let log_lines = + String.split logs ~on:'\n' + |> List.filter ~f:(String.is_prefix ~prefix:"{\"timestamp\":") + in + let jsons = List.map log_lines ~f:Yojson.Safe.from_string in + let metadata_jsons = + List.map jsons ~f:(fun json -> + match json with + | `Assoc items -> ( + match List.Assoc.find items ~equal:String.equal "metadata" with + | Some md -> + md + | None -> + failwithf "Log line is missing metadata: %s" + (Yojson.Safe.to_string json) + () ) + | other -> + failwithf "Expected log line to be a JSON record, got: %s" + (Yojson.Safe.to_string other) + () ) + in + let state_hash_and_blocks = + List.fold metadata_jsons ~init:[] ~f:(fun acc json -> + match json with + | `Assoc items -> ( + match + List.Assoc.find items ~equal:String.equal "precomputed_block" + with + | Some block -> ( + match + List.Assoc.find items ~equal:String.equal "state_hash" + with + | Some state_hash -> + (state_hash, block) :: acc + | None -> + failwith + "Log metadata contains a precomputed block, but no \ + state hash" ) + | None -> + acc ) + | other -> + failwithf "Expected log line to be a JSON record, got: %s" + (Yojson.Safe.to_string other) + () ) + in + let%bind.Deferred () = + Deferred.List.iter state_hash_and_blocks + ~f:(fun (state_hash_json, block_json) -> + let double_quoted_state_hash = + Yojson.Safe.to_string state_hash_json + in + let state_hash = + String.sub double_quoted_state_hash ~pos:1 + ~len:(String.length double_quoted_state_hash - 2) + in + let block = Yojson.Safe.pretty_to_string block_json in + let filename = state_hash ^ ".json" in + match%map.Deferred Sys.file_exists filename with + | `Yes -> + [%log info] + "File already exists for precomputed block with state hash %s" + state_hash + | _ -> + [%log info] + "Dumping precomputed block with state hash %s to file %s" + state_hash filename ; + Out_channel.with_file filename ~f:(fun out_ch -> + Out_channel.output_string out_ch block ) ) + in + Malleable_error.return () + + let get_metrics ~logger t = + let open Deferred.Or_error.Let_syntax in + [%log info] "Getting node's metrics" ~metadata:(logger_metadata t) ; + let query_obj = Graphql.Query_metrics.make () in + let%bind query_result_obj = + exec_graphql_request ~logger ~node:t ~query_name:"query_metrics" query_obj + in + [%log info] "get_metrics, finished exec_graphql_request" ; + let block_production_delay = + Array.to_list + @@ query_result_obj.daemonStatus.metrics.blockProductionDelay + in + let metrics = query_result_obj.daemonStatus.metrics in + let transaction_pool_diff_received = metrics.transactionPoolDiffReceived in + let transaction_pool_diff_broadcasted = + metrics.transactionPoolDiffBroadcasted + in + let transactions_added_to_pool = metrics.transactionsAddedToPool in + let transaction_pool_size = metrics.transactionPoolSize in + [%log info] + "get_metrics, result of graphql query (block_production_delay; \ + tx_received; tx_broadcasted; txs_added_to_pool; tx_pool_size) (%s; %d; \ + %d; %d; %d)" + ( String.concat ~sep:", " + @@ List.map ~f:string_of_int block_production_delay ) + transaction_pool_diff_received transaction_pool_diff_broadcasted + transactions_added_to_pool transaction_pool_size ; + return + Intf. + { block_production_delay + ; transaction_pool_diff_broadcasted + ; transaction_pool_diff_received + ; transactions_added_to_pool + ; transaction_pool_size + } +end + +module Workload_to_deploy = struct + type t = { workload_id : string; pod_info : Node.pod_info } + + let construct_workload workload_id pod_info : t = { workload_id; pod_info } + + let cons_pod_info ?network_keypair ?(has_archive_container = false) + primary_container_id : Node.pod_info = + { network_keypair; has_archive_container; primary_container_id } + + let get_nodes_from_workload t ~config = + let%bind cwd = Unix.getcwd () in + let open Malleable_error.Let_syntax in + let%bind app_id = + Deferred.bind ~f:Malleable_error.or_hard_error + (Integration_test_lib.Util.run_cmd_or_error cwd "kubectl" + ( base_kube_args config + @ [ "get" + ; "deployment" + ; t.workload_id + ; "-o" + ; "jsonpath={.spec.selector.matchLabels.app}" + ] ) ) + in + let%map pod_ids_str = + Integration_test_lib.Util.run_cmd_or_hard_error cwd "kubectl" + ( base_kube_args config + @ [ "get"; "pod"; "-l"; "app=" ^ app_id; "-o"; "name" ] ) + in + let pod_ids = + String.split pod_ids_str ~on:'\n' + |> List.filter ~f:(Fn.compose not String.is_empty) + |> List.map ~f:(String.substr_replace_first ~pattern:"pod/" ~with_:"") + in + (* we have a strict 1 workload to 1 pod setup, except the snark workers. *) + (* elsewhere in the code I'm simply using List.hd_exn which is not ideal but enabled by the fact that in all relevant cases, there's only going to be 1 pod id in pod_ids *) + (* TODO fix this^ and have a more elegant solution *) + let pod_info = t.pod_info in + { Node.app_id; pod_ids; pod_info; config } +end + +type t = + { namespace : string + ; constants : Test_config.constants + ; seeds : Node.t Core.String.Map.t + ; block_producers : Node.t Core.String.Map.t + ; snark_coordinators : Node.t Core.String.Map.t + ; snark_workers : Node.t Core.String.Map.t + ; archive_nodes : Node.t Core.String.Map.t + (* ; nodes_by_pod_id : Node.t Core.String.Map.t *) + ; testnet_log_filter : string + ; genesis_keypairs : Network_keypair.t Core.String.Map.t + } + +let constants { constants; _ } = constants + +let constraint_constants { constants; _ } = constants.constraints + +let genesis_constants { constants; _ } = constants.genesis + +let seeds { seeds; _ } = seeds + +let block_producers { block_producers; _ } = block_producers + +let snark_coordinators { snark_coordinators; _ } = snark_coordinators + +let snark_workers { snark_workers; _ } = snark_workers + +let archive_nodes { archive_nodes; _ } = archive_nodes + +(* all_nodes returns all *actual* mina nodes; note that a snark_worker is a pod within the network but not technically a mina node, therefore not included here. snark coordinators on the other hand ARE mina nodes *) +let all_nodes { seeds; block_producers; snark_coordinators; archive_nodes; _ } = + List.concat + [ Core.String.Map.to_alist seeds + ; Core.String.Map.to_alist block_producers + ; Core.String.Map.to_alist snark_coordinators + ; Core.String.Map.to_alist archive_nodes + ] + |> Core.String.Map.of_alist_exn + +(* all_pods returns everything in the network. remember that snark_workers will never initialize and will never sync, and aren't supposed to *) +(* TODO snark workers and snark coordinators have the same key name, but different workload ids*) +let all_pods t = + List.concat + [ Core.String.Map.to_alist t.seeds + ; Core.String.Map.to_alist t.block_producers + ; Core.String.Map.to_alist t.snark_coordinators + ; Core.String.Map.to_alist t.snark_workers + ; Core.String.Map.to_alist t.archive_nodes + ] + |> Core.String.Map.of_alist_exn + +(* all_non_seed_pods returns everything in the network except seed nodes *) +let all_non_seed_pods t = + List.concat + [ Core.String.Map.to_alist t.block_producers + ; Core.String.Map.to_alist t.snark_coordinators + ; Core.String.Map.to_alist t.snark_workers + ; Core.String.Map.to_alist t.archive_nodes + ] + |> Core.String.Map.of_alist_exn + +let genesis_keypairs { genesis_keypairs; _ } = genesis_keypairs + +let lookup_node_by_pod_id t id = + let pods = all_pods t |> Core.Map.to_alist in + List.fold pods ~init:None ~f:(fun acc (node_name, node) -> + match acc with + | Some acc -> + Some acc + | None -> + if String.equal id (List.hd_exn node.pod_ids) then + Some (node_name, node) + else None ) + +let all_pod_ids t = + let pods = all_pods t |> Core.Map.to_alist in + List.fold pods ~init:[] ~f:(fun acc (_, node) -> + List.cons (List.hd_exn node.pod_ids) acc ) + +let initialize_infra ~logger network = + let open Malleable_error.Let_syntax in + let poll_interval = Time.Span.of_sec 15.0 in + let max_polls = 40 (* 10 mins *) in + let all_pods_set = all_pod_ids network |> String.Set.of_list in + let kube_get_pods () = + Integration_test_lib.Util.run_cmd_or_error_timeout ~timeout_seconds:60 "/" + "kubectl" + [ "-n" + ; network.namespace + ; "get" + ; "pods" + ; "-ojsonpath={range \ + .items[*]}{.metadata.name}{':'}{.status.phase}{'\\n'}{end}" + ] + in + let parse_pod_statuses result_str = + result_str |> String.split_lines + |> List.map ~f:(fun line -> + let parts = String.split line ~on:':' in + assert (Mina_stdlib.List.Length.Compare.(parts = 2)) ; + (List.nth_exn parts 0, List.nth_exn parts 1) ) + |> List.filter ~f:(fun (pod_name, _) -> + String.Set.mem all_pods_set pod_name ) + (* this filters out the archive bootstrap pods, since they aren't in all_pods_set. in fact the bootstrap pods aren't tracked at all in the framework *) + |> String.Map.of_alist_exn + in + let rec poll n = + [%log debug] "Checking kubernetes pod statuses, n=%d" n ; + let is_successful_pod_status status = String.equal "Running" status in + match%bind Deferred.bind ~f:Malleable_error.return (kube_get_pods ()) with + | Ok str -> + let pod_statuses = parse_pod_statuses str in + [%log debug] "pod_statuses: \n %s" + ( String.Map.to_alist pod_statuses + |> List.map ~f:(fun (key, data) -> key ^ ": " ^ data ^ "\n") + |> String.concat ) ; + [%log debug] "all_pods: \n %s" + (String.Set.elements all_pods_set |> String.concat ~sep:", ") ; + let all_pods_are_present = + List.for_all (String.Set.elements all_pods_set) ~f:(fun pod_id -> + String.Map.mem pod_statuses pod_id ) + in + let any_pods_are_not_running = + (* there could be duplicate keys... *) + List.exists + (String.Map.data pod_statuses) + ~f:(Fn.compose not is_successful_pod_status) + in + if not all_pods_are_present then ( + let present_pods = String.Map.keys pod_statuses in + [%log fatal] + "Not all pods were found when querying namespace; this indicates a \ + deployment error. Refusing to continue. \n\ + Expected pods: [%s]. \n\ + Present pods: [%s]" + (String.Set.elements all_pods_set |> String.concat ~sep:"; ") + (present_pods |> String.concat ~sep:"; ") ; + Malleable_error.hard_error_string ~exit_code:5 + "Some pods were not found in namespace." ) + else if any_pods_are_not_running then + let failed_pod_statuses = + List.filter (String.Map.to_alist pod_statuses) + ~f:(fun (_, status) -> not (is_successful_pod_status status)) + in + if n > 0 then ( + [%log debug] "Got bad pod statuses, polling again ($failed_statuses" + ~metadata: + [ ( "failed_statuses" + , `Assoc + (List.Assoc.map failed_pod_statuses ~f:(fun v -> + `String v ) ) ) + ] ; + let%bind () = + after poll_interval |> Deferred.bind ~f:Malleable_error.return + in + poll (n - 1) ) + else ( + [%log fatal] + "Got bad pod statuses, not all pods were assigned to nodes and \ + ready in time. pod statuses: ($failed_statuses" + ~metadata: + [ ( "failed_statuses" + , `Assoc + (List.Assoc.map failed_pod_statuses ~f:(fun v -> + `String v ) ) ) + ] ; + Malleable_error.hard_error_string ~exit_code:4 + "Some pods either were not assigned to nodes or did not deploy \ + properly." ) + else return () + | Error _ -> + [%log debug] "`kubectl get pods` timed out, polling again" ; + let%bind () = + after poll_interval |> Deferred.bind ~f:Malleable_error.return + in + poll n + in + [%log info] "Waiting for pods to be assigned nodes and become ready" ; + let res = poll max_polls in + match%bind.Deferred res with + | Error _ -> + [%log error] "Not all pods were assigned nodes, cannot proceed!" ; + res + | Ok _ -> + [%log info] "Pods assigned to nodes" ; + res From 5d67e4cc8bf2a7abbeb5ecc95a9375d6a7a1ec0a Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 25 Jul 2023 16:51:45 +0000 Subject: [PATCH 05/90] Fix formatting --- .../abstract_network.ml | 17 ++- .../ci_interaction.ml | 121 +++++++++--------- 2 files changed, 71 insertions(+), 67 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index 51f1bae74a6..baf76de978a 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -1064,14 +1064,19 @@ module Node = struct (* return an error if [t] isn't an archive node *) let dump_archive_data ~logger (t : t) ~data_file = let%bind data = - match%map send_ci_http_request ~access_token:t.config.access_token ~request_body:(Request.GetNodeLogs (id t)) with - | Response.NodeLogs (_, logs) -> logs - | _ -> failwith "invalid node logs response" + match%map + send_ci_http_request ~access_token:t.config.access_token + ~request_body:(Request.Get_node_logs (id t)) + with + | Ok (Response.Node_logs (_, logs)) -> + logs + | _ -> + failwith "invalid node logs response" in [%log info] "Dumping archive data to file %s" data_file ; - Malleable_error.return @@ - Out_channel.with_file data_file ~f:(fun out_ch -> - Out_channel.output_string out_ch data ) + Malleable_error.return + @@ Out_channel.with_file data_file ~f:(fun out_ch -> + Out_channel.output_string out_ch data ) let run_replayer ~logger (t : t) = [%log info] "Running replayer on archived data (node: %s, container: %s)" diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 7ce324a9027..10496934275 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -21,7 +21,7 @@ let cluster_zone = "us-west1a" module Network_config = struct module Cli_inputs = Cli_inputs - + type block_producer_config = { name : string; keypair : Network_keypair.t; libp2p_secret : string } [@@deriving to_yojson] @@ -71,15 +71,14 @@ module Network_config = struct fun map -> `Assoc (Core.Map.fold_right ~init:[] - ~f:(fun ~key:k ~data:v accum -> - (k, Network_keypair.to_yojson v) :: accum ) - map )] ) + ~f:(fun ~key:k ~data:v accum -> + (k, Network_keypair.to_yojson v) :: accum ) + map )] ) ; constants : Test_config.constants ; terraform : terraform_config } [@@deriving to_yojson] - let terraform_config_to_assoc t = let[@warning "-8"] (`Assoc assoc : Yojson.Safe.t) = terraform_config_to_yojson t @@ -121,7 +120,7 @@ module Network_config = struct if List.contains_dup ~compare:String.compare key_names_list then failwith "All accounts in genesis ledger must have unique names. Check to make \ - sure you are not using the same account_name more than once" ; + sure you are not using the same account_name more than once" ; let all_nodes_names_list = List.map block_producers ~f:(fun acct -> acct.node_name) @ match snark_coordinator with None -> [] | Some n -> [ n.node_name ] @@ -129,15 +128,15 @@ module Network_config = struct if List.contains_dup ~compare:String.compare all_nodes_names_list then failwith "All nodes in testnet must have unique names. Check to make sure you \ - are not using the same node_name more than once" ; + are not using the same node_name more than once" ; (* GENERATE ACCOUNTS AND KEYPAIRS *) let keypairs = List.take (* the first keypair is the genesis winner and is assumed to be untimed. Therefore dropping it, and not assigning it to any block producer *) (List.drop - (Array.to_list (Lazy.force Key_gen.Sample_keypairs.keypairs)) - 1 ) + (Array.to_list (Lazy.force Key_gen.Sample_keypairs.keypairs)) + 1 ) (List.length genesis_ledger) in let labeled_accounts : @@ -220,9 +219,9 @@ module Network_config = struct { base = Accounts (let tuplist = String.Map.data genesis_ledger_accounts in - List.map tuplist ~f:(fun tup -> - let acct, _ = tup in - acct ) ) + List.map tuplist ~f:(fun tup -> + let acct, _ = tup in + acct ) ) ; add_genesis_winner = None ; num_accounts = None ; balances = [] @@ -235,7 +234,7 @@ module Network_config = struct let genesis_constants = Or_error.ok_exn (Genesis_ledger_helper.make_genesis_constants ~logger - ~default:Genesis_constants.compiled runtime_config ) + ~default:Genesis_constants.compiled runtime_config ) in let constants : Test_config.constants = { constraints = constraint_constants; genesis = genesis_constants } @@ -260,8 +259,8 @@ module Network_config = struct let failstring = Format.sprintf "Failing because the account key of all initial block \ - producers must be in the genesis ledger. name of Node: \ - %s. name of Account which does not exist: %s" + producers must be in the genesis ledger. name of Node: \ + %s. name of Account which does not exist: %s" node.node_name node.account_name in failwith failstring @@ -288,9 +287,9 @@ module Network_config = struct let genesis_keypairs = String.Map.of_alist_exn (List.map (String.Map.to_alist genesis_ledger_accounts) - ~f:(fun element -> - let kp_name, (_, (pk, sk)) = element in - (kp_name, mk_net_keypair kp_name (pk, sk)) ) ) + ~f:(fun element -> + let kp_name, (_, (pk, sk)) = element in + (kp_name, mk_net_keypair kp_name (pk, sk)) ) ) in let snark_coordinator_config = match snark_coordinator with @@ -305,8 +304,8 @@ module Network_config = struct let failstring = Format.sprintf "Failing because the account key of all initial snark \ - coordinators must be in the genesis ledger. name of \ - Node: %s. name of Account which does not exist: %s" + coordinators must be in the genesis ledger. name of \ + Node: %s. name of Account which does not exist: %s" node.node_name node.account_name in failwith failstring @@ -404,10 +403,10 @@ end module Request = struct type t = | Access_token - | CreateNetwork of Network_config.t - | DeployNetwork of Network_id.t - | DestroyNetwork of Network_id.t - | GetNodeLogs of Node_id.t + | Create_network of Network_config.t + | Deploy_network of Network_id.t + | Destroy_network of Network_id.t + | Get_node_logs of Node_id.t [@@deriving to_yojson] end @@ -420,11 +419,9 @@ module Node_type = struct | Snark_coordinator [@@deriving eq, yojson] - let to_string nt = - to_yojson nt |> Yojson.Safe.to_string + let to_string nt = to_yojson nt |> Yojson.Safe.to_string - let of_string s = - of_yojson @@ `List [ `String s ] |> Result.ok_or_failwith + let of_string s = of_yojson @@ `List [ `String s ] |> Result.ok_or_failwith end module Network_deploy_response = struct @@ -454,10 +451,10 @@ end module Response = struct type t = | Access_token of Access_token.t - | NetworkCreated of Network_id.t - | NetworkDeployed of Network_deploy_response.t - | NetworkDestroyed - | NodeLogs of Node_id.t * string + | Network_created of Network_id.t + | Network_deployed of Network_deploy_response.t + | Network_destroyed + | Node_logs of Node_id.t * string [@@deriving eq, of_yojson] end @@ -466,58 +463,64 @@ end https://www.notion.so/minafoundation/Lucy-CI-Interactions-e36b48ac52994cafbe1367548e02241d?pvs=4 *) -let request_ci_access_token () : Response.t Async.Deferred.t = +let request_ci_access_token () : Response.t Deferred.Or_error.t = failwith "request_ci_access_token" (* for example, we can communicate with the CI via https and test-specific access token *) -let[@warning "-27"] send_ci_http_request ~(access_token: Access_token.t) ~(request_body: Request.t) : Response.t Async.Deferred.t = +let[@warning "-27"] send_ci_http_request ~(access_token : Access_token.t) + ~(request_body : Request.t) : Response.t Deferred.Or_error.t = let req_str = request_body |> Request.to_yojson |> Yojson.Safe.to_string in failwithf "send_ci_http_request: %s\n" req_str () module Request_unit_tests = struct - let%test_unit "Create network request" = assert true (* TODO: too complicated for now *) + let%test_unit "Create network request" = assert true + (* TODO: too complicated for now *) let%test_unit "Deploy network request" = let open Request in - let result = DeployNetwork "network0" |> to_yojson |> Yojson.Safe.to_string in + let result = + Deploy_network "network0" |> to_yojson |> Yojson.Safe.to_string + in let ( = ) = String.equal in - assert (result = {| [ "DeployNetwork", "network0" ] |}) + assert (result = {|["Deploy_network","network0"]|}) let%test_unit "Destroy network request" = let open Request in - let result = DestroyNetwork "network0" |> to_yojson |> Yojson.Safe.to_string in + let result = + Destroy_network "network0" |> to_yojson |> Yojson.Safe.to_string + in let ( = ) = String.equal in - assert (result = {| [ "DestroyNetwork", "network0" ] |}) + assert (result = {|["Destroy_network","network0"]|}) let%test_unit "Get node logs request" = let open Request in - let result = GetNodeLogs "node0" |> to_yojson |> Yojson.Safe.to_string in + let result = Get_node_logs "node0" |> to_yojson |> Yojson.Safe.to_string in let ( = ) = String.equal in - assert (result = {| [ "GetNodeLogs", "node0" ] |}) + assert (result = {|["Get_node_logs","node0"]|}) end module Response_unit_tests = struct let%test_unit "Parse network created response" = - let open Response in + let open Response in let result = - `List [ `String "NetworkCreated"; `String "node0" ] + `List [ `String "Network_created"; `String "node0" ] |> of_yojson |> Result.ok_or_failwith in let ( = ) = equal in - assert ( - result = NetworkCreated "node0" - ) + assert (result = Network_created "node0") let%test_unit "Parse network deployed response" = let open Node_type in let open Network_deploy_response in let result = - {| { "node0": ("Archive_node", "gql0"), - "node1": ("Block_producer_node", "gql1"), - "node2": ("Non_seed_node", "gql2"), - "node3": ("Seed_node", "gql3"), - "node4": ("Snark_coordinator", "gql4") - } |} + {| + { "node0": ("Archive_node", "gql0") + , "node1": ("Block_producer_node", "gql1") + , "node2": ("Non_seed_node", "gql2") + , "node3": ("Seed_node", "gql3") + , "node4": ("Snark_coordinator", "gql4") + } + |} |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith in let ( = ) = equal in @@ -534,21 +537,17 @@ module Response_unit_tests = struct let%test_unit "Parse network destroyed response" = let open Response in let result = - `List [ `String "NetworkDestroyed" ] + `List [ `String "Network_destroyed" ] |> of_yojson |> Result.ok_or_failwith in - assert ( - equal result NetworkDestroyed - ) + assert (equal result Network_destroyed) let%test_unit "Parse node logs response" = let open Response in let result = - `List [ `String "NodeLogs"; `String "node0"; `String "node0_logs" ] + `List [ `String "Node_logs"; `String "node0"; `String "node0_logs" ] |> of_yojson |> Result.ok_or_failwith in let ( = ) = equal in - assert ( - result = NodeLogs ("node0", "node0_logs") - ) -end \ No newline at end of file + assert (result = Node_logs ("node0", "node0_logs")) +end From 071e8656e885198f9f0611642dfaeba9475886df Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 26 Jul 2023 04:22:06 +0000 Subject: [PATCH 06/90] Pull abstract engine into test_executive --- src/app/test_executive/dune | 2 +- src/app/test_executive/test_executive.ml | 14 +------ .../cli_inputs.ml | 42 ++----------------- 3 files changed, 6 insertions(+), 52 deletions(-) diff --git a/src/app/test_executive/dune b/src/app/test_executive/dune index 2f2bdad97b6..b637a93c718 100644 --- a/src/app/test_executive/dune +++ b/src/app/test_executive/dune @@ -36,7 +36,7 @@ currency mina_runtime_config secrets - integration_test_cloud_engine + integration_test_abstract_engine mina_generators logger random_oracle diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index ba6b2d3f9f6..2e9e283a605 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -47,7 +47,7 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : else Deferred.return () let engines : engine list = - [ ("cloud", (module Integration_test_cloud_engine : Intf.Engine.S)) ] + [ ("cloud", (module Integration_test_abstract_engine : Intf.Engine.S)) ] let tests : test list = [ ( "peers-reliability" @@ -241,7 +241,6 @@ let dispatch_cleanup ~logger ~pause_cleanup_func ~network_cleanup_func deferred let main inputs = - (* TODO: abstract over which engine is in use, allow engine to be set form CLI *) let (Test_inputs_with_cli_inputs ((module Test_inputs), cli_inputs)) = inputs.test_inputs in @@ -253,16 +252,6 @@ let main inputs = and type node = Engine.Network.Node.t and type dsl = Dsl.t ) in - (* - (module Test (Test_inputs) - : Intf.Test.S - with type network = Engine.Network.t - and type log_engine = Engine.Log_engine.t ) - *) - (* TODO: - * let (module Exec) = (module Execute.Make (Engine)) in - * Exec.execute ~logger ~engine_cli_inputs ~images (module Test (Engine)) - *) let logger = Logger.create () in let images = { Test_config.Container_images.mina = inputs.mina_image @@ -360,7 +349,6 @@ let main inputs = in [%log trace] "initializing network abstraction" ; let%bind () = Engine.Network.initialize_infra ~logger network in - [%log info] "Starting the daemons within the pods" ; let start_print (node : Engine.Network.Node.t) = let open Malleable_error.Let_syntax in diff --git a/src/lib/integration_test_abstract_engine/cli_inputs.ml b/src/lib/integration_test_abstract_engine/cli_inputs.ml index 7bcc7a257aa..388ab7b317d 100644 --- a/src/lib/integration_test_abstract_engine/cli_inputs.ml +++ b/src/lib/integration_test_abstract_engine/cli_inputs.ml @@ -1,11 +1,9 @@ open Cmdliner +(* TODO: finalize this *) + type t = - { mina_automation_location : string - ; check_capacity : bool - ; check_capacity_delay : int - ; check_capacity_retries : int - } + { mina_automation_location : string } let term = let mina_automation_location = @@ -20,36 +18,4 @@ let term = [ "mina-automation-location" ] ~env ~docv:"MINA_AUTOMATION_LOCATION" ~doc) in - let check_capacity = - let doc = - "Whether or not to check the capacity of the cloud cluster before \ - execution. Default: true" - in - Arg.(value & opt bool true & info [ "capacity-check" ] ~doc) - in - let check_capacity_delay = - let doc = - "Upon a failed capacity check, how much time in seconds to wait before \ - trying. Only holds meaning if check-capacity is true. Default: 60" - in - Arg.(value & opt int 60 & info [ "capacity-check-delay" ] ~doc) - in - let check_capacity_retries = - let doc = - "Upon a failed capacity check, how many times to retry before giving \ - up. Only holds meaning if check-capacity is true. Default: 10" - in - Arg.(value & opt int 10 & info [ "capacity-check-retries" ] ~doc) - in - let cons_inputs mina_automation_location check_capacity check_capacity_delay - check_capacity_retries = - { mina_automation_location - ; check_capacity - ; check_capacity_delay - ; check_capacity_retries - } - in - - Term.( - const cons_inputs $ mina_automation_location $ check_capacity - $ check_capacity_delay $ check_capacity_retries) + mina_automation_location From 75b5b2317a685b08103b62abab2ae5bc459a44ac Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 26 Jul 2023 04:23:13 +0000 Subject: [PATCH 07/90] Separate platform-specific and platform-agnostic actions --- .../ci_interaction.ml | 220 ++++++++++++++---- 1 file changed, 172 insertions(+), 48 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 10496934275..14433e6dbfa 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -401,21 +401,36 @@ module Node_id = struct end module Request = struct - type t = - | Access_token - | Create_network of Network_config.t - | Deploy_network of Network_id.t - | Destroy_network of Network_id.t - | Get_node_logs of Node_id.t - [@@deriving to_yojson] + module Platform_specific = struct + type t = + | Create_network of Network_config.t + | Deploy_network of Network_id.t + | Destroy_network of Network_id.t + [@@deriving to_yojson] + end + + type start_node = { node_id : Node_id.t; fresh_state : bool; commit_sha : string } + [@@deriving eq, yojson] + + module Platform_agnostic = struct + type t = + | Access_token + | Start_node of start_node + | Stop_node of Node_id.t + | Dump_archive_data of Node_id.t + | Dump_mina_logs of Node_id.t + | Dump_precomputed_blocks of Node_id.t + | Run_replayer of Node_id.t + [@@deriving to_yojson] + end end module Node_type = struct type t = | Archive_node | Block_producer_node - | Non_seed_node | Seed_node + | Snark_worker | Snark_coordinator [@@deriving eq, yojson] @@ -428,12 +443,14 @@ module Network_deploy_response = struct module Map = Map.Make (String) include Map - type t = (Node_type.t * string) Map.t [@@deriving eq] + type t = (Node_type.t * string option) Map.t [@@deriving eq] let of_yojson = let f accum = function | node_id, `Tuple [ `String nt; `String gql_uri ] -> - Map.set accum ~key:node_id ~data:(Node_type.of_string nt, gql_uri) + Map.set accum ~key:node_id ~data:(Node_type.of_string nt, Some gql_uri) + | node_id, `Tuple [ `String nt; `Null ] -> + Map.set accum ~key:node_id ~data:(Node_type.of_string nt, None) | _ -> failwith "invalid network_deploy_response yojson entry" in @@ -449,13 +466,35 @@ module Access_token = struct end module Response = struct - type t = - | Access_token of Access_token.t - | Network_created of Network_id.t - | Network_deployed of Network_deploy_response.t - | Network_destroyed - | Node_logs of Node_id.t * string - [@@deriving eq, of_yojson] + module Platform_specific = struct + type t = + | Network_created of Network_id.t + | Network_deployed of Network_deploy_response.t + | Network_destroyed + [@@deriving eq, of_yojson] + end + + module Node_logs = struct + type t = string [@@deriving eq] + + let to_yojson s = `String s + + let of_yojson = function + | `String s -> Ok s + | x -> Error (Yojson.Safe.to_string x) + end + + module Platform_agnostic = struct + type t = + | Access_token of Access_token.t + | Node_started of Node_id.t + | Node_stopped of Node_id.t + | Archive_data_dump of Node_id.t * string + | Mina_logs_dump of Node_id.t * Node_logs.t + | Precomputed_block_dump of Node_id.t * string + | Replayer_run of Node_id.t * string + [@@deriving eq, of_yojson] + end end (** @@ -463,50 +502,91 @@ end https://www.notion.so/minafoundation/Lucy-CI-Interactions-e36b48ac52994cafbe1367548e02241d?pvs=4 *) -let request_ci_access_token () : Response.t Deferred.Or_error.t = +let request_ci_access_token () : Response.Platform_agnostic.t Deferred.Or_error.t = failwith "request_ci_access_token" (* for example, we can communicate with the CI via https and test-specific access token *) let[@warning "-27"] send_ci_http_request ~(access_token : Access_token.t) - ~(request_body : Request.t) : Response.t Deferred.Or_error.t = - let req_str = request_body |> Request.to_yojson |> Yojson.Safe.to_string in + ~(request_body : Request.Platform_specific.t) : Response.Platform_specific.t Deferred.Or_error.t = + let req_str = request_body |> Request.Platform_specific.to_yojson |> Yojson.Safe.to_string in + (* TODO: http request *) + failwithf "send_ci_http_request: %s\n" req_str () + +let[@warning "-27"] send_ci_http_request' ~(access_token : Access_token.t) + ~(request_body : Request.Platform_agnostic.t) : Response.Platform_agnostic.t Deferred.Or_error.t = + let req_str = request_body |> Request.Platform_agnostic.to_yojson |> Yojson.Safe.to_string in + (* TODO: http request *) failwithf "send_ci_http_request: %s\n" req_str () module Request_unit_tests = struct - let%test_unit "Create network request" = assert true - (* TODO: too complicated for now *) + let ( = ) = String.equal + + let%test_unit "Create network request" = + assert true + (* TODO: too complicated for now *) let%test_unit "Deploy network request" = - let open Request in + let open Request.Platform_specific in let result = Deploy_network "network0" |> to_yojson |> Yojson.Safe.to_string in - let ( = ) = String.equal in assert (result = {|["Deploy_network","network0"]|}) let%test_unit "Destroy network request" = - let open Request in + let open Request.Platform_specific in let result = Destroy_network "network0" |> to_yojson |> Yojson.Safe.to_string in - let ( = ) = String.equal in assert (result = {|["Destroy_network","network0"]|}) - let%test_unit "Get node logs request" = - let open Request in - let result = Get_node_logs "node0" |> to_yojson |> Yojson.Safe.to_string in - let ( = ) = String.equal in - assert (result = {|["Get_node_logs","node0"]|}) + let%test_unit "Access token request" = + let open Request.Platform_agnostic in + let result = Access_token |> to_yojson |> Yojson.Safe.to_string in + assert (result = {|["Access_token"]|}) + + let%test_unit "Start node request" = + let open Request.Platform_agnostic in + let result = Start_node { node_id = "node0"; fresh_state = true; commit_sha = "0123456" } + |> to_yojson |> Yojson.Safe.to_string + in + assert (result = {|["Start_node",{"node_id":"node0","fresh_state":true,"commit_sha":"0123456"}]|}) + + let%test_unit "Stop node request" = + let open Request.Platform_agnostic in + let result = Stop_node "node0" |> to_yojson |> Yojson.Safe.to_string in + assert (result = {|["Stop_node","node0"]|}) + + let%test_unit "Dump archive data request" = + let open Request.Platform_agnostic in + let result = Dump_archive_data "node0" |> to_yojson |> Yojson.Safe.to_string in + assert (result = {|["Dump_archive_data","node0"]|}) + + let%test_unit "Dump mina logs request" = + let open Request.Platform_agnostic in + let result = Dump_mina_logs "node0" |> to_yojson |> Yojson.Safe.to_string in + assert (result = {|["Dump_mina_logs","node0"]|}) + + let%test_unit "Dump precomputed blocks request" = + let open Request.Platform_agnostic in + let result = Dump_precomputed_blocks "node0" |> to_yojson |> Yojson.Safe.to_string in + assert (result = {|["Dump_precomputed_blocks","node0"]|}) + + let%test_unit "Run replayer request" = + let open Request.Platform_agnostic in + let result = Run_replayer "node0" |> to_yojson |> Yojson.Safe.to_string in + assert (result = {|["Run_replayer","node0"]|}) end module Response_unit_tests = struct + open Response.Platform_specific + + let ( = ) = equal + let%test_unit "Parse network created response" = - let open Response in let result = `List [ `String "Network_created"; `String "node0" ] |> of_yojson |> Result.ok_or_failwith in - let ( = ) = equal in assert (result = Network_created "node0") let%test_unit "Parse network deployed response" = @@ -516,8 +596,8 @@ module Response_unit_tests = struct {| { "node0": ("Archive_node", "gql0") , "node1": ("Block_producer_node", "gql1") - , "node2": ("Non_seed_node", "gql2") - , "node3": ("Seed_node", "gql3") + , "node2": ("Seed_node", "gql2") + , "node3": ("Snark_worker", null) , "node4": ("Snark_coordinator", "gql4") } |} @@ -527,27 +607,71 @@ module Response_unit_tests = struct assert ( result = of_alist_exn - [ ("node0", (Archive_node, "gql0")) - ; ("node1", (Block_producer_node, "gql1")) - ; ("node2", (Non_seed_node, "gql2")) - ; ("node3", (Seed_node, "gql3")) - ; ("node4", (Snark_coordinator, "gql4")) + [ ("node0", (Archive_node, Some "gql0")) + ; ("node1", (Block_producer_node, Some "gql1")) + ; ("node2", (Seed_node, Some "gql2")) + ; ("node3", (Snark_worker, None)) + ; ("node4", (Snark_coordinator, Some "gql4")) ] ) let%test_unit "Parse network destroyed response" = - let open Response in let result = - `List [ `String "Network_destroyed" ] + {|["Network_destroyed"]|} |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith in - assert (equal result Network_destroyed) + assert (result = Network_destroyed) + + open Response.Platform_agnostic + + let ( = ) = equal - let%test_unit "Parse node logs response" = - let open Response in + let%test_unit "Parse access token response" = let result = - `List [ `String "Node_logs"; `String "node0"; `String "node0_logs" ] + {|["Access_token","token0"]|} |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith in - let ( = ) = equal in - assert (result = Node_logs ("node0", "node0_logs")) + assert (result = Access_token "token0") + + let%test_unit "Node started response" = + let result = + {|["Node_started","node0"]|} |> Yojson.Safe.from_string + |> of_yojson |> Result.ok_or_failwith + in + assert (result = Node_started "node0") + + let%test_unit "Node stopped response" = + let result = + {|["Node_stopped","node0"]|} |> Yojson.Safe.from_string + |> of_yojson |> Result.ok_or_failwith + in + assert (result = Node_stopped "node0") + + let%test_unit "Archive data dump response" = + let result = + {|["Archive_data_dump","node0","data0"]|} |> Yojson.Safe.from_string + |> of_yojson |> Result.ok_or_failwith + in + assert (result = Archive_data_dump ("node0", "data0")) + + let%test_unit "Mina logs dump response" = + let raw_logs = "{\"log0\":\"msg0\"}\n{\"log1\":\"msg1\"}" in + let result = + `List [ `String "Mina_logs_dump"; `String "node0"; `String raw_logs ] + |> of_yojson |> Result.ok_or_failwith + in + assert (result = Mina_logs_dump ("node0", raw_logs)) + + let%test_unit "Precomputed block dump response" = + let result = + {|["Precomputed_block_dump","node0","blocks"]|} |> Yojson.Safe.from_string + |> of_yojson |> Result.ok_or_failwith + in + assert (result = Precomputed_block_dump ("node0", "blocks")) + + let%test_unit "Replayer run response" = + let result = + {|["Replayer_run","node0","logs0"]|} |> Yojson.Safe.from_string + |> of_yojson |> Result.ok_or_failwith + in + assert (result = Replayer_run ("node0", "logs0")) end From d5c03c9e93c41403a282d2cb41741759e4e360d9 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 26 Jul 2023 04:24:44 +0000 Subject: [PATCH 08/90] Continue adding CI stubs --- .../abstract_network.ml | 213 +++++++----------- .../mina_automation.ml | 7 +- src/lib/integration_test_lib/intf.ml | 5 +- 3 files changed, 85 insertions(+), 140 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index baf76de978a..16d5764ea63 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -7,6 +7,7 @@ open Ci_interaction (* exclude from bisect_ppx to avoid type error on GraphQL modules *) [@@@coverage exclude_file] +(* TODO: remove *) let mina_archive_container_id = "mina_archive_container_id" let mina_archive_username = "mina_archive_username" @@ -25,110 +26,77 @@ type config = ; namespace : string ; graphql_enabled : bool ; access_token : Access_token.t + ; network_id : Network_id.t + ; ingress_uri : string + ; current_commit_sha : string } +(* TODO: remove *) let base_kube_args { cluster; namespace; _ } = [ "--cluster"; cluster; "--namespace"; namespace ] -let id : Network_id.t Ivar.t = Async.Ivar.create () - -let network_ingress_uri : string Ivar.t = Async.Ivar.create () - module Node = struct - type pod_info = - { network_keypair : Network_keypair.t option - ; primary_container_id : string - (* this is going to be probably either "mina" or "worker" *) - ; has_archive_container : bool - (* archive pods have a "mina" container and an "archive" container alongside *) - } - + (* TODO: remove app_id, add ingress_uri *) type t = { app_id : string - ; pod_ids : string list - ; pod_info : pod_info + ; node_id : string + ; network_keypair : Network_keypair.t option ; config : config + ; node_type : Node_type.t } - let id { pod_ids; _ } = List.hd_exn pod_ids + let id { node_id; _ } = node_id - let network_keypair { pod_info = { network_keypair; _ }; _ } = network_keypair + let network_keypair { network_keypair; _ } = network_keypair + (* TODO: remove *) let base_kube_args t = [ "--cluster"; t.cluster; "--namespace"; t.namespace ] - let get_logs_in_container ?container_id { pod_ids; config; pod_info; _ } = + (* TODO: remove *) + let get_logs_in_container ?container_id { node_id; config; _ } = let container_id = - Option.value container_id ~default:pod_info.primary_container_id + Option.value container_id ~default:"" in let%bind cwd = Unix.getcwd () in Integration_test_lib.Util.run_cmd_or_hard_error ~exit_code:13 cwd "kubectl" ( base_kube_args config - @ [ "logs"; "-c"; container_id; List.hd_exn pod_ids ] ) - - let run_in_container ?(exit_code = 10) ?container_id ?override_with_pod_id - ~cmd t = - let { config; pod_info; _ } = t in - let pod_id = - match override_with_pod_id with - | Some pid -> - pid - | None -> - List.hd_exn t.pod_ids - in - let container_id = - Option.value container_id ~default:pod_info.primary_container_id - in - let%bind cwd = Unix.getcwd () in - Integration_test_lib.Util.run_cmd_or_hard_error ~exit_code cwd "kubectl" - ( base_kube_args config - @ [ "exec"; "-c"; container_id; "-i"; pod_id; "--" ] - @ cmd ) + @ [ "logs"; "-c"; container_id; node_id ] ) - let cp_string_to_container_file ?container_id ~str ~dest t = - let { pod_ids; config; pod_info; _ } = t in - let container_id = - Option.value container_id ~default:pod_info.primary_container_id - in - let tmp_file, oc = - Caml.Filename.open_temp_file ~temp_dir:Filename.temp_dir_name - "integration_test_cp_string" ".tmp" - in - Out_channel.output_string oc str ; - Out_channel.close oc ; - let%bind cwd = Unix.getcwd () in - let dest_file = - sprintf "%s/%s:%s" config.namespace (List.hd_exn pod_ids) dest + let start ?commit_sha ~fresh_state t : unit Malleable_error.t = + let commit_sha = Option.value commit_sha ~default:t.config.current_commit_sha in + let access_token = t.config.access_token in + let request_body = Request.Platform_agnostic.Start_node { node_id = id t; fresh_state; commit_sha } in + let%bind () = + match%map send_ci_http_request' ~access_token ~request_body with + | Ok (Response.Platform_agnostic.Node_started node_id) when String.equal node_id (id t) -> () + | _ -> failwith "invalid node started response" in - Integration_test_lib.Util.run_cmd_or_error cwd "kubectl" - (base_kube_args config @ [ "cp"; "-c"; container_id; tmp_file; dest_file ]) + Malleable_error.return () - let start ~fresh_state node : unit Malleable_error.t = - let open Malleable_error.Let_syntax in + let stop t = + let access_token = t.config.access_token in + let request_body = Request.Platform_agnostic.Stop_node (id t) in let%bind () = - if fresh_state then - run_in_container node ~cmd:[ "sh"; "-c"; "rm -rf .mina-config/*" ] - >>| ignore - else Malleable_error.return () + match%map send_ci_http_request' ~access_token ~request_body with + | Ok (Response.Platform_agnostic.Node_stopped node_id) when String.equal node_id (id t) -> () + | _ -> failwith "invalid node stopped response" in - run_in_container ~exit_code:11 node ~cmd:[ "/start.sh" ] >>| ignore - - let stop node = - let open Malleable_error.Let_syntax in - run_in_container ~exit_code:12 node ~cmd:[ "/stop.sh" ] >>| ignore + Malleable_error.return () let logger_metadata node = [ ("namespace", `String node.config.namespace) ; ("app_id", `String node.app_id) - ; ("pod_id", `String (List.hd_exn node.pod_ids)) + ; ("node_id", `String (node.node_id)) ] module Scalars = Graphql_lib.Scalars module Graphql = struct + (* TODO: fix *) let ingress_uri node = let host = sprintf "%s.%s" node.config.testnet_name - (Ivar.value_exn network_ingress_uri) + node.config.ingress_uri in let path = sprintf "/%s/graphql" node.app_id in Uri.make ~scheme:"http" ~host ~path ~port:80 () @@ -1060,18 +1028,18 @@ module Node = struct else Deferred.Or_error.error_string "Node is not currently capturing structured log messages" - - (* return an error if [t] isn't an archive node *) + + (* TODO: check if node is an archive node *) let dump_archive_data ~logger (t : t) ~data_file = let%bind data = match%map - send_ci_http_request ~access_token:t.config.access_token - ~request_body:(Request.Get_node_logs (id t)) + send_ci_http_request' ~access_token:t.config.access_token + ~request_body:(Request.Platform_agnostic.Dump_archive_data (id t)) with - | Ok (Response.Node_logs (_, logs)) -> + | Ok (Response.Platform_agnostic.Archive_data_dump (node_id, logs)) when String.equal node_id (id t) -> logs | _ -> - failwith "invalid node logs response" + failwith "invalid archive dump data response" in [%log info] "Dumping archive data to file %s" data_file ; Malleable_error.return @@ -1079,50 +1047,33 @@ module Node = struct Out_channel.output_string out_ch data ) let run_replayer ~logger (t : t) = - [%log info] "Running replayer on archived data (node: %s, container: %s)" - (List.hd_exn t.pod_ids) mina_archive_container_id ; - let open Malleable_error.Let_syntax in - let%bind accounts = - run_in_container t - ~cmd:[ "jq"; "-c"; ".ledger.accounts"; "/root/config/daemon.json" ] + [%log info] "Running replayer on archived data node: %s" + (t.node_id) ; + let access_token = t.config.access_token in + let request_body = Request.Platform_agnostic.Run_replayer (id t) in + let%bind output = + match%map send_ci_http_request' ~access_token ~request_body with + | Ok (Response.Platform_agnostic.Replayer_run (node_id, output)) when String.equal node_id (id t) -> + output + | _ -> failwith "invalid run replayer response" in - let replayer_input = - sprintf - {| { "genesis_ledger": { "accounts": %s, "add_genesis_winner": true }} |} - accounts - in - let dest = "replayer-input.json" in - let%bind _res = - Deferred.bind ~f:Malleable_error.return - (cp_string_to_container_file t ~container_id:mina_archive_container_id - ~str:replayer_input ~dest ) - in - run_in_container t ~container_id:mina_archive_container_id - ~cmd: - [ "mina-replayer" - ; "--archive-uri" - ; postgres_url - ; "--input-file" - ; dest - ; "--output-file" - ; "/dev/null" - ; "--continue-on-error" - ] + Malleable_error.return output + (* TODO: convert *) let dump_mina_logs ~logger (t : t) ~log_file = let open Malleable_error.Let_syntax in - [%log info] "Dumping container logs from (node: %s, container: %s)" - (List.hd_exn t.pod_ids) t.pod_info.primary_container_id ; + [%log info] "Dumping logs from node: %s" + (t.node_id) ; let%map logs = get_logs_in_container t in [%log info] "Dumping container log to file %s" log_file ; Out_channel.with_file log_file ~f:(fun out_ch -> Out_channel.output_string out_ch logs ) + (* TODO: convert *) let dump_precomputed_blocks ~logger (t : t) = let open Malleable_error.Let_syntax in [%log info] - "Dumping precomputed blocks from logs for (node: %s, container: %s)" - (List.hd_exn t.pod_ids) t.pod_info.primary_container_id ; + "Dumping precomputed blocks from logs for node: %s" t.node_id ; let%bind logs = get_logs_in_container t in (* kubectl logs may include non-log output, like "Using password from environment variable" *) let log_lines = @@ -1234,13 +1185,13 @@ module Node = struct end module Workload_to_deploy = struct - type t = { workload_id : string; pod_info : Node.pod_info } + type t = { workload_id : string; pod_info : string } let construct_workload workload_id pod_info : t = { workload_id; pod_info } - let cons_pod_info ?network_keypair ?(has_archive_container = false) - primary_container_id : Node.pod_info = - { network_keypair; has_archive_container; primary_container_id } + let[@warning "-27"] cons_pod_info ?network_keypair ?(has_archive_container = false) + primary_container_id = "" + (* { network_keypair; has_archive_container; primary_container_id } *) let get_nodes_from_workload t ~config = let%bind cwd = Unix.getcwd () in @@ -1256,36 +1207,36 @@ module Workload_to_deploy = struct ; "jsonpath={.spec.selector.matchLabels.app}" ] ) ) in - let%map pod_ids_str = + let%map node_id_str = Integration_test_lib.Util.run_cmd_or_hard_error cwd "kubectl" ( base_kube_args config @ [ "get"; "pod"; "-l"; "app=" ^ app_id; "-o"; "name" ] ) in - let pod_ids = - String.split pod_ids_str ~on:'\n' + let node_id = + String.split node_id_str ~on:'\n' |> List.filter ~f:(Fn.compose not String.is_empty) |> List.map ~f:(String.substr_replace_first ~pattern:"pod/" ~with_:"") + |> String.concat ~sep:"TODO:" in - (* we have a strict 1 workload to 1 pod setup, except the snark workers. *) - (* elsewhere in the code I'm simply using List.hd_exn which is not ideal but enabled by the fact that in all relevant cases, there's only going to be 1 pod id in pod_ids *) - (* TODO fix this^ and have a more elegant solution *) - let pod_info = t.pod_info in - { Node.app_id; pod_ids; pod_info; config } + let node_type = Node_type.Seed_node in + let network_keypair = None in + { Node.app_id; node_id; node_type; config; network_keypair } end type t = - { namespace : string - ; constants : Test_config.constants + { constants : Test_config.constants ; seeds : Node.t Core.String.Map.t ; block_producers : Node.t Core.String.Map.t ; snark_coordinators : Node.t Core.String.Map.t ; snark_workers : Node.t Core.String.Map.t ; archive_nodes : Node.t Core.String.Map.t - (* ; nodes_by_pod_id : Node.t Core.String.Map.t *) ; testnet_log_filter : string ; genesis_keypairs : Network_keypair.t Core.String.Map.t + ; network_id : Network_id.t } +let id { network_id; _ } = network_id + let constants { constants; _ } = constants let constraint_constants { constants; _ } = constants.constraints @@ -1313,7 +1264,7 @@ let all_nodes { seeds; block_producers; snark_coordinators; archive_nodes; _ } = |> Core.String.Map.of_alist_exn (* all_pods returns everything in the network. remember that snark_workers will never initialize and will never sync, and aren't supposed to *) -(* TODO snark workers and snark coordinators have the same key name, but different workload ids*) +(* TODO: snark workers and snark coordinators have the same key name, but different workload ids*) let all_pods t = List.concat [ Core.String.Map.to_alist t.seeds @@ -1336,32 +1287,22 @@ let all_non_seed_pods t = let genesis_keypairs { genesis_keypairs; _ } = genesis_keypairs -let lookup_node_by_pod_id t id = - let pods = all_pods t |> Core.Map.to_alist in - List.fold pods ~init:None ~f:(fun acc (node_name, node) -> - match acc with - | Some acc -> - Some acc - | None -> - if String.equal id (List.hd_exn node.pod_ids) then - Some (node_name, node) - else None ) - -let all_pod_ids t = +let all_node_id t = let pods = all_pods t |> Core.Map.to_alist in List.fold pods ~init:[] ~f:(fun acc (_, node) -> - List.cons (List.hd_exn node.pod_ids) acc ) + List.cons (node.node_id) acc ) +(* TODO: what to do with this? *) let initialize_infra ~logger network = let open Malleable_error.Let_syntax in let poll_interval = Time.Span.of_sec 15.0 in let max_polls = 40 (* 10 mins *) in - let all_pods_set = all_pod_ids network |> String.Set.of_list in + let all_pods_set = all_node_id network |> String.Set.of_list in let kube_get_pods () = Integration_test_lib.Util.run_cmd_or_error_timeout ~timeout_seconds:60 "/" "kubectl" [ "-n" - ; network.namespace + ; "namespace" ; "get" ; "pods" ; "-ojsonpath={range \ diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index 2a082479dc2..e89c6e9cd5d 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -342,6 +342,9 @@ module Network_manager = struct ; namespace = t.namespace ; graphql_enabled = t.graphql_enabled ; access_token = "access_token" (* TODO: *) + ; network_id = "network_id" (* TODO: *) + ; ingress_uri = "ingress_uri" (* TODO: *) + ; current_commit_sha = "0000000" (* TODO: *) } in let func_for_fold ~(key : string) ~data accum_M = @@ -373,13 +376,13 @@ module Network_manager = struct ~f:func_for_fold in let network = - { Abstract_network.namespace = t.namespace + { Abstract_network.network_id = t.namespace (* TODO: fix *) ; constants = t.constants ; seeds ; block_producers ; snark_coordinators ; snark_workers - ; archive_nodes (* ; all_nodes *) + ; archive_nodes ; testnet_log_filter = t.testnet_log_filter ; genesis_keypairs = t.genesis_keypairs } diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index 5977f4f4c28..4ff32738248 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -53,7 +53,8 @@ module Engine = struct val network_keypair : t -> Network_keypair.t option - val start : fresh_state:bool -> t -> unit Malleable_error.t + val start : + ?commit_sha:string -> fresh_state:bool -> t -> unit Malleable_error.t val stop : t -> unit Malleable_error.t @@ -104,7 +105,7 @@ module Engine = struct val initialize_infra : logger:Logger.t -> t -> unit Malleable_error.t - val id : string Async.Ivar.t + val id : t -> string end module type Network_manager_intf = sig From 3eaea43c6951bcac5c2efd13321defa1ddbb147f Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 26 Jul 2023 04:25:41 +0000 Subject: [PATCH 09/90] Reformat --- .../abstract_network.ml | 65 ++++++++++-------- .../ci_interaction.ml | 67 ++++++++++++------- .../cli_inputs.ml | 3 +- 3 files changed, 79 insertions(+), 56 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index 16d5764ea63..11c8f6d7fc4 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -54,22 +54,27 @@ module Node = struct (* TODO: remove *) let get_logs_in_container ?container_id { node_id; config; _ } = - let container_id = - Option.value container_id ~default:"" - in + let container_id = Option.value container_id ~default:"" in let%bind cwd = Unix.getcwd () in Integration_test_lib.Util.run_cmd_or_hard_error ~exit_code:13 cwd "kubectl" - ( base_kube_args config - @ [ "logs"; "-c"; container_id; node_id ] ) + (base_kube_args config @ [ "logs"; "-c"; container_id; node_id ]) let start ?commit_sha ~fresh_state t : unit Malleable_error.t = - let commit_sha = Option.value commit_sha ~default:t.config.current_commit_sha in + let commit_sha = + Option.value commit_sha ~default:t.config.current_commit_sha + in let access_token = t.config.access_token in - let request_body = Request.Platform_agnostic.Start_node { node_id = id t; fresh_state; commit_sha } in + let request_body = + Request.Platform_agnostic.Start_node + { node_id = id t; fresh_state; commit_sha } + in let%bind () = match%map send_ci_http_request' ~access_token ~request_body with - | Ok (Response.Platform_agnostic.Node_started node_id) when String.equal node_id (id t) -> () - | _ -> failwith "invalid node started response" + | Ok (Response.Platform_agnostic.Node_started node_id) + when String.equal node_id (id t) -> + () + | _ -> + failwith "invalid node started response" in Malleable_error.return () @@ -78,15 +83,18 @@ module Node = struct let request_body = Request.Platform_agnostic.Stop_node (id t) in let%bind () = match%map send_ci_http_request' ~access_token ~request_body with - | Ok (Response.Platform_agnostic.Node_stopped node_id) when String.equal node_id (id t) -> () - | _ -> failwith "invalid node stopped response" + | Ok (Response.Platform_agnostic.Node_stopped node_id) + when String.equal node_id (id t) -> + () + | _ -> + failwith "invalid node stopped response" in Malleable_error.return () let logger_metadata node = [ ("namespace", `String node.config.namespace) ; ("app_id", `String node.app_id) - ; ("node_id", `String (node.node_id)) + ; ("node_id", `String node.node_id) ] module Scalars = Graphql_lib.Scalars @@ -95,8 +103,7 @@ module Node = struct (* TODO: fix *) let ingress_uri node = let host = - sprintf "%s.%s" node.config.testnet_name - node.config.ingress_uri + sprintf "%s.%s" node.config.testnet_name node.config.ingress_uri in let path = sprintf "/%s/graphql" node.app_id in Uri.make ~scheme:"http" ~host ~path ~port:80 () @@ -1028,7 +1035,7 @@ module Node = struct else Deferred.Or_error.error_string "Node is not currently capturing structured log messages" - + (* TODO: check if node is an archive node *) let dump_archive_data ~logger (t : t) ~data_file = let%bind data = @@ -1036,7 +1043,8 @@ module Node = struct send_ci_http_request' ~access_token:t.config.access_token ~request_body:(Request.Platform_agnostic.Dump_archive_data (id t)) with - | Ok (Response.Platform_agnostic.Archive_data_dump (node_id, logs)) when String.equal node_id (id t) -> + | Ok (Response.Platform_agnostic.Archive_data_dump (node_id, logs)) + when String.equal node_id (id t) -> logs | _ -> failwith "invalid archive dump data response" @@ -1047,23 +1055,23 @@ module Node = struct Out_channel.output_string out_ch data ) let run_replayer ~logger (t : t) = - [%log info] "Running replayer on archived data node: %s" - (t.node_id) ; + [%log info] "Running replayer on archived data node: %s" t.node_id ; let access_token = t.config.access_token in let request_body = Request.Platform_agnostic.Run_replayer (id t) in let%bind output = match%map send_ci_http_request' ~access_token ~request_body with - | Ok (Response.Platform_agnostic.Replayer_run (node_id, output)) when String.equal node_id (id t) -> + | Ok (Response.Platform_agnostic.Replayer_run (node_id, output)) + when String.equal node_id (id t) -> output - | _ -> failwith "invalid run replayer response" + | _ -> + failwith "invalid run replayer response" in Malleable_error.return output (* TODO: convert *) let dump_mina_logs ~logger (t : t) ~log_file = let open Malleable_error.Let_syntax in - [%log info] "Dumping logs from node: %s" - (t.node_id) ; + [%log info] "Dumping logs from node: %s" t.node_id ; let%map logs = get_logs_in_container t in [%log info] "Dumping container log to file %s" log_file ; Out_channel.with_file log_file ~f:(fun out_ch -> @@ -1072,8 +1080,7 @@ module Node = struct (* TODO: convert *) let dump_precomputed_blocks ~logger (t : t) = let open Malleable_error.Let_syntax in - [%log info] - "Dumping precomputed blocks from logs for node: %s" t.node_id ; + [%log info] "Dumping precomputed blocks from logs for node: %s" t.node_id ; let%bind logs = get_logs_in_container t in (* kubectl logs may include non-log output, like "Using password from environment variable" *) let log_lines = @@ -1189,9 +1196,10 @@ module Workload_to_deploy = struct let construct_workload workload_id pod_info : t = { workload_id; pod_info } - let[@warning "-27"] cons_pod_info ?network_keypair ?(has_archive_container = false) - primary_container_id = "" - (* { network_keypair; has_archive_container; primary_container_id } *) + let[@warning "-27"] cons_pod_info ?network_keypair + ?(has_archive_container = false) primary_container_id = + "" + (* { network_keypair; has_archive_container; primary_container_id } *) let get_nodes_from_workload t ~config = let%bind cwd = Unix.getcwd () in @@ -1289,8 +1297,7 @@ let genesis_keypairs { genesis_keypairs; _ } = genesis_keypairs let all_node_id t = let pods = all_pods t |> Core.Map.to_alist in - List.fold pods ~init:[] ~f:(fun acc (_, node) -> - List.cons (node.node_id) acc ) + List.fold pods ~init:[] ~f:(fun acc (_, node) -> List.cons node.node_id acc) (* TODO: what to do with this? *) let initialize_infra ~logger network = diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 14433e6dbfa..1b1e8a63b1a 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -409,8 +409,9 @@ module Request = struct [@@deriving to_yojson] end - type start_node = { node_id : Node_id.t; fresh_state : bool; commit_sha : string } - [@@deriving eq, yojson] + type start_node = + { node_id : Node_id.t; fresh_state : bool; commit_sha : string } + [@@deriving eq, yojson] module Platform_agnostic = struct type t = @@ -473,15 +474,17 @@ module Response = struct | Network_destroyed [@@deriving eq, of_yojson] end - + module Node_logs = struct type t = string [@@deriving eq] let to_yojson s = `String s let of_yojson = function - | `String s -> Ok s - | x -> Error (Yojson.Safe.to_string x) + | `String s -> + Ok s + | x -> + Error (Yojson.Safe.to_string x) end module Platform_agnostic = struct @@ -502,28 +505,34 @@ end https://www.notion.so/minafoundation/Lucy-CI-Interactions-e36b48ac52994cafbe1367548e02241d?pvs=4 *) -let request_ci_access_token () : Response.Platform_agnostic.t Deferred.Or_error.t = +let request_ci_access_token () : + Response.Platform_agnostic.t Deferred.Or_error.t = failwith "request_ci_access_token" (* for example, we can communicate with the CI via https and test-specific access token *) let[@warning "-27"] send_ci_http_request ~(access_token : Access_token.t) - ~(request_body : Request.Platform_specific.t) : Response.Platform_specific.t Deferred.Or_error.t = - let req_str = request_body |> Request.Platform_specific.to_yojson |> Yojson.Safe.to_string in + ~(request_body : Request.Platform_specific.t) : + Response.Platform_specific.t Deferred.Or_error.t = + let req_str = + request_body |> Request.Platform_specific.to_yojson |> Yojson.Safe.to_string + in (* TODO: http request *) failwithf "send_ci_http_request: %s\n" req_str () let[@warning "-27"] send_ci_http_request' ~(access_token : Access_token.t) - ~(request_body : Request.Platform_agnostic.t) : Response.Platform_agnostic.t Deferred.Or_error.t = - let req_str = request_body |> Request.Platform_agnostic.to_yojson |> Yojson.Safe.to_string in + ~(request_body : Request.Platform_agnostic.t) : + Response.Platform_agnostic.t Deferred.Or_error.t = + let req_str = + request_body |> Request.Platform_agnostic.to_yojson |> Yojson.Safe.to_string + in (* TODO: http request *) failwithf "send_ci_http_request: %s\n" req_str () module Request_unit_tests = struct let ( = ) = String.equal - let%test_unit "Create network request" = - assert true - (* TODO: too complicated for now *) + let%test_unit "Create network request" = assert true + (* TODO: too complicated for now *) let%test_unit "Deploy network request" = let open Request.Platform_specific in @@ -546,10 +555,14 @@ module Request_unit_tests = struct let%test_unit "Start node request" = let open Request.Platform_agnostic in - let result = Start_node { node_id = "node0"; fresh_state = true; commit_sha = "0123456" } + let result = + Start_node + { node_id = "node0"; fresh_state = true; commit_sha = "0123456" } |> to_yojson |> Yojson.Safe.to_string in - assert (result = {|["Start_node",{"node_id":"node0","fresh_state":true,"commit_sha":"0123456"}]|}) + assert ( + result + = {|["Start_node",{"node_id":"node0","fresh_state":true,"commit_sha":"0123456"}]|} ) let%test_unit "Stop node request" = let open Request.Platform_agnostic in @@ -558,7 +571,9 @@ module Request_unit_tests = struct let%test_unit "Dump archive data request" = let open Request.Platform_agnostic in - let result = Dump_archive_data "node0" |> to_yojson |> Yojson.Safe.to_string in + let result = + Dump_archive_data "node0" |> to_yojson |> Yojson.Safe.to_string + in assert (result = {|["Dump_archive_data","node0"]|}) let%test_unit "Dump mina logs request" = @@ -568,7 +583,9 @@ module Request_unit_tests = struct let%test_unit "Dump precomputed blocks request" = let open Request.Platform_agnostic in - let result = Dump_precomputed_blocks "node0" |> to_yojson |> Yojson.Safe.to_string in + let result = + Dump_precomputed_blocks "node0" |> to_yojson |> Yojson.Safe.to_string + in assert (result = {|["Dump_precomputed_blocks","node0"]|}) let%test_unit "Run replayer request" = @@ -616,8 +633,8 @@ module Response_unit_tests = struct let%test_unit "Parse network destroyed response" = let result = - {|["Network_destroyed"]|} |> Yojson.Safe.from_string - |> of_yojson |> Result.ok_or_failwith + {|["Network_destroyed"]|} |> Yojson.Safe.from_string |> of_yojson + |> Result.ok_or_failwith in assert (result = Network_destroyed) @@ -627,22 +644,22 @@ module Response_unit_tests = struct let%test_unit "Parse access token response" = let result = - {|["Access_token","token0"]|} |> Yojson.Safe.from_string - |> of_yojson |> Result.ok_or_failwith + {|["Access_token","token0"]|} |> Yojson.Safe.from_string |> of_yojson + |> Result.ok_or_failwith in assert (result = Access_token "token0") let%test_unit "Node started response" = let result = - {|["Node_started","node0"]|} |> Yojson.Safe.from_string - |> of_yojson |> Result.ok_or_failwith + {|["Node_started","node0"]|} |> Yojson.Safe.from_string |> of_yojson + |> Result.ok_or_failwith in assert (result = Node_started "node0") let%test_unit "Node stopped response" = let result = - {|["Node_stopped","node0"]|} |> Yojson.Safe.from_string - |> of_yojson |> Result.ok_or_failwith + {|["Node_stopped","node0"]|} |> Yojson.Safe.from_string |> of_yojson + |> Result.ok_or_failwith in assert (result = Node_stopped "node0") diff --git a/src/lib/integration_test_abstract_engine/cli_inputs.ml b/src/lib/integration_test_abstract_engine/cli_inputs.ml index 388ab7b317d..cc66fdb8f79 100644 --- a/src/lib/integration_test_abstract_engine/cli_inputs.ml +++ b/src/lib/integration_test_abstract_engine/cli_inputs.ml @@ -2,8 +2,7 @@ open Cmdliner (* TODO: finalize this *) -type t = - { mina_automation_location : string } +type t = { mina_automation_location : string } let term = let mina_automation_location = From b8a7015ab729f800827e9f5a0c600909728f72c3 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Thu, 27 Jul 2023 03:17:54 +0000 Subject: [PATCH 10/90] Remove capacity check --- .../integration_test_abstract_engine/mina_automation.ml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index e89c6e9cd5d..89eed4e6817 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -284,14 +284,6 @@ module Network_manager = struct ; genesis_keypairs = network_config.genesis_keypairs } in - (* check capacity *) - let%bind () = - if network_config.check_capacity then - check_kube_capacity t ~logger - ~delay:(Float.of_int network_config.check_capacity_delay) - ~retries:network_config.check_capacity_retries - else Malleable_error.return () - in (* making the main.tf.json *) let open Deferred.Let_syntax in let%bind () = From 4e268ae7b30300a2492a0484055611195707e4ff Mon Sep 17 00:00:00 2001 From: Quantifier Date: Thu, 27 Jul 2023 03:32:17 +0000 Subject: [PATCH 11/90] Implement http requests --- .../ci_interaction.ml | 112 ++++++++++++++---- src/lib/integration_test_abstract_engine/dune | 4 +- 2 files changed, 92 insertions(+), 24 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 1b1e8a63b1a..61109918c68 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -59,12 +59,10 @@ module Network_config = struct } [@@deriving to_yojson] + (* TODO: replace with t' *) type t = { mina_automation_location : string ; debug_arg : bool - ; check_capacity : bool - ; check_capacity_delay : int - ; check_capacity_retries : int ; genesis_keypairs : (Network_keypair.t Core.String.Map.t [@to_yojson @@ -79,6 +77,23 @@ module Network_config = struct } [@@deriving to_yojson] + (* TODO: remove *) + type t' = + { mina_automation_location : string + ; debug_arg : bool + ; genesis_keypairs : + (Network_keypair.t Core.String.Map.t + [@to_yojson + fun map -> + `Assoc + (Core.Map.fold_right ~init:[] + ~f:(fun ~key:k ~data:v accum -> + (k, Network_keypair.to_yojson v) :: accum ) + map )] ) + ; constants : Test_config.constants + } + [@@deriving to_yojson] + let terraform_config_to_assoc t = let[@warning "-8"] (`Assoc assoc : Yojson.Safe.t) = terraform_config_to_yojson t @@ -322,9 +337,6 @@ module Network_config = struct (* NETWORK CONFIG *) { mina_automation_location = cli_inputs.mina_automation_location ; debug_arg = debug - ; check_capacity = cli_inputs.check_capacity - ; check_capacity_delay = cli_inputs.check_capacity_delay - ; check_capacity_retries = cli_inputs.check_capacity_retries ; genesis_keypairs ; constants ; terraform = @@ -403,7 +415,7 @@ end module Request = struct module Platform_specific = struct type t = - | Create_network of Network_config.t + | Create_network of Network_config.t' | Deploy_network of Network_id.t | Destroy_network of Network_id.t [@@deriving to_yojson] @@ -424,6 +436,10 @@ module Request = struct | Run_replayer of Node_id.t [@@deriving to_yojson] end + + module type S = sig + type t [@@deriving to_yojson] + end end module Node_type = struct @@ -475,6 +491,7 @@ module Response = struct [@@deriving eq, of_yojson] end + (** Logproc takes care of these logs so we bypass them here *) module Node_logs = struct type t = string [@@deriving eq] @@ -498,6 +515,10 @@ module Response = struct | Replayer_run of Node_id.t * string [@@deriving eq, of_yojson] end + + module type S = sig + type t [@@deriving of_yojson] + end end (** @@ -505,28 +526,73 @@ end https://www.notion.so/minafoundation/Lucy-CI-Interactions-e36b48ac52994cafbe1367548e02241d?pvs=4 *) +module Ci = struct + type t = { host : string; port : int } [@@deriving of_yojson] + + let ci : t Async.Ivar.t = Async.Ivar.create () + + let uri () = + let ci = Async.Ivar.value_exn ci in + Uri.make ~scheme:"http" ~host:ci.host ~port:ci.port () +end + let request_ci_access_token () : Response.Platform_agnostic.t Deferred.Or_error.t = failwith "request_ci_access_token" -(* for example, we can communicate with the CI via https and test-specific access token *) -let[@warning "-27"] send_ci_http_request ~(access_token : Access_token.t) - ~(request_body : Request.Platform_specific.t) : - Response.Platform_specific.t Deferred.Or_error.t = - let req_str = - request_body |> Request.Platform_specific.to_yojson |> Yojson.Safe.to_string +let post ~(access_token : Access_token.t) ~request_body = + let headers = + let open Cohttp in + Header.add_list (Header.init ()) + [ ("Authorization", access_token) + ; ("Accept", "application/json") + ; ("Content-Type", "application/json") + ] in - (* TODO: http request *) - failwithf "send_ci_http_request: %s\n" req_str () - -let[@warning "-27"] send_ci_http_request' ~(access_token : Access_token.t) - ~(request_body : Request.Platform_agnostic.t) : - Response.Platform_agnostic.t Deferred.Or_error.t = - let req_str = - request_body |> Request.Platform_agnostic.to_yojson |> Yojson.Safe.to_string + Deferred.Or_error.try_with ~here:[%here] ~extract_exn:true (fun () -> + Cohttp_async.Client.post ~headers + ~body:(Cohttp_async.Body.of_string request_body) + (Ci.uri ()) ) + +let to_json ~response ~body_str = + match Cohttp.Code.code_of_status (Cohttp_async.Response.status response) with + | 200 -> + Deferred.return (Ok (Yojson.Safe.from_string body_str)) + | code -> + Deferred.return + (Error (Error.createf "Status code %d -- %s" code body_str)) + +let http (type a b) (module Req : Request.S with type t = a) + (module Res : Response.S with type t = b) ~access_token ~(request_body : a) + = + let request_body = Req.to_yojson request_body |> Yojson.Safe.to_string in + let open Deferred.Or_error.Let_syntax in + let%bind response, body = post ~access_token ~request_body in + let%bind body_str = + Cohttp_async.Body.to_string body |> Deferred.map ~f:Result.return in - (* TODO: http request *) - failwithf "send_ci_http_request: %s\n" req_str () + let%bind body_json = to_json ~response ~body_str in + let open Yojson.Safe.Util in + ( match (member "errors" body_json, member "data" body_json) with + | `Null, `Null -> + Error (Error.of_string "Empty response from http query") + | error, `Null -> + Error (Error.of_string @@ to_string error) + | _, raw_json -> + Res.of_yojson raw_json |> Result.map_error ~f:Error.of_string ) + |> Deferred.return + +let ci_http_request ~access_token ~request_body = + http + (module Request.Platform_specific) + (module Response.Platform_specific) + ~access_token ~request_body + +let ci_http_request' ~access_token ~request_body = + http + (module Request.Platform_agnostic) + (module Response.Platform_agnostic) + ~access_token ~request_body module Request_unit_tests = struct let ( = ) = String.equal diff --git a/src/lib/integration_test_abstract_engine/dune b/src/lib/integration_test_abstract_engine/dune index 64fd802203a..0f7072c8875 100644 --- a/src/lib/integration_test_abstract_engine/dune +++ b/src/lib/integration_test_abstract_engine/dune @@ -7,7 +7,7 @@ (preprocess (pps ppx_mina ppx_version ppx_let ppx_pipebang ppx_inline_test ppx_custom_printf ppx_deriving_yojson lens.ppx_deriving - ppx_sexp_conv ppx_assert ppx_deriving.eq + ppx_sexp_conv ppx_assert ppx_deriving.eq ppx_here graphql_ppx -- %{read-lines:../../graphql-ppx-config.inc})) (libraries ;; opam libraries @@ -15,6 +15,8 @@ async_kernel core_kernel core + cohttp + cohttp-async async cmdliner base From 860eb4d1b8e5382e9789ba6973d81fe20f4b8bf2 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Thu, 27 Jul 2023 03:33:49 +0000 Subject: [PATCH 12/90] Unit test network config json --- .../ci_interaction.ml | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 61109918c68..7986a26cd9f 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -597,8 +597,25 @@ let ci_http_request' ~access_token ~request_body = module Request_unit_tests = struct let ( = ) = String.equal - let%test_unit "Create network request" = assert true - (* TODO: too complicated for now *) + let%test_unit "Create network request" = + let constants : Test_config.constants = + { constraints = Genesis_constants.Constraint_constants.compiled + ; genesis = Genesis_constants.compiled + } + in + let network_config : Network_config.t' = + { mina_automation_location = "loc" + ; debug_arg = true + ; genesis_keypairs = Core.String.Map.empty + ; constants + } + in + let result = + Network_config.t'_to_yojson network_config |> Yojson.Safe.to_string + in + assert ( + result + = {|{"mina_automation_location":"loc","debug_arg":true,"genesis_keypairs":{},"constants":{"constraints":{"sub_windows_per_window":11,"ledger_depth":35,"work_delay":2,"block_window_duration_ms":180000,"transaction_capacity_log_2":7,"pending_coinbase_depth":5,"coinbase_amount":"720000000000","supercharged_coinbase_factor":1,"account_creation_fee":"1","fork":null},"genesis":{"protocol":{"k":290,"slots_per_epoch":7140,"slots_per_sub_window":7,"delta":0,"genesis_state_timestamp":"2020-09-16 10:15:00.000000Z"},"txpool_max_size":3000,"num_accounts":null,"zkapp_proof_update_cost":10.26,"zkapp_signed_single_update_cost":9.140000000000001,"zkapp_signed_pair_update_cost":10.08,"zkapp_transaction_cost_limit":69.45,"max_event_elements":100,"max_action_elements":100}}}|} ) let%test_unit "Deploy network request" = let open Request.Platform_specific in From 8df3795bda51b945626a0d49dc9d7b7df893f3a2 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Thu, 27 Jul 2023 03:34:51 +0000 Subject: [PATCH 13/90] Continue abstracting the node functions --- .../abstract_network.ml | 274 +++++++++--------- .../cli_inputs.ml | 3 +- src/lib/integration_test_lib/intf.ml | 2 - 3 files changed, 143 insertions(+), 136 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index 11c8f6d7fc4..b430770712d 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -7,19 +7,6 @@ open Ci_interaction (* exclude from bisect_ppx to avoid type error on GraphQL modules *) [@@@coverage exclude_file] -(* TODO: remove *) -let mina_archive_container_id = "mina_archive_container_id" - -let mina_archive_username = "mina_archive_username" - -let mina_archive_pw = "mina_archive_pw" - -let postgres_url = - Printf.sprintf "postgres://%s:%s@archive-1-postgresql:5432/archive" - mina_archive_username mina_archive_pw - -let node_password = "naughty blue worm" - type config = { testnet_name : string ; cluster : string @@ -31,10 +18,6 @@ type config = ; current_commit_sha : string } -(* TODO: remove *) -let base_kube_args { cluster; namespace; _ } = - [ "--cluster"; cluster; "--namespace"; namespace ] - module Node = struct (* TODO: remove app_id, add ingress_uri *) type t = @@ -43,6 +26,7 @@ module Node = struct ; network_keypair : Network_keypair.t option ; config : config ; node_type : Node_type.t + ; password : string } let id { node_id; _ } = node_id @@ -69,7 +53,7 @@ module Node = struct { node_id = id t; fresh_state; commit_sha } in let%bind () = - match%map send_ci_http_request' ~access_token ~request_body with + match%map ci_http_request' ~access_token ~request_body with | Ok (Response.Platform_agnostic.Node_started node_id) when String.equal node_id (id t) -> () @@ -82,7 +66,7 @@ module Node = struct let access_token = t.config.access_token in let request_body = Request.Platform_agnostic.Stop_node (id t) in let%bind () = - match%map send_ci_http_request' ~access_token ~request_body with + match%map ci_http_request' ~access_token ~request_body with | Ok (Response.Platform_agnostic.Node_stopped node_id) when String.equal node_id (id t) -> () @@ -718,7 +702,7 @@ module Node = struct let unlock_account_obj = Graphql.Unlock_account.( make - @@ makeVariables ~password:node_password ~public_key:sender_pub_key ()) + @@ makeVariables ~password:t.password ~public_key:sender_pub_key ()) in exec_graphql_request ~logger ~node:t ~initial_delay_sec:0. ~query_name:"unlock_sender_account_graphql" unlock_account_obj @@ -853,8 +837,7 @@ module Node = struct let unlock_account_obj = Graphql.Unlock_account.( make - @@ makeVariables ~password:"naughty blue worm" - ~public_key:sender_pub_key ()) + @@ makeVariables ~password:t.password ~public_key:sender_pub_key ()) in exec_graphql_request ~logger ~node:t ~query_name:"unlock_sender_account_graphql" unlock_account_obj @@ -1036,123 +1019,140 @@ module Node = struct Deferred.Or_error.error_string "Node is not currently capturing structured log messages" - (* TODO: check if node is an archive node *) let dump_archive_data ~logger (t : t) ~data_file = - let%bind data = - match%map - send_ci_http_request' ~access_token:t.config.access_token - ~request_body:(Request.Platform_agnostic.Dump_archive_data (id t)) - with - | Ok (Response.Platform_agnostic.Archive_data_dump (node_id, logs)) - when String.equal node_id (id t) -> - logs - | _ -> - failwith "invalid archive dump data response" - in - [%log info] "Dumping archive data to file %s" data_file ; - Malleable_error.return - @@ Out_channel.with_file data_file ~f:(fun out_ch -> - Out_channel.output_string out_ch data ) + if Node_type.(equal t.node_type Archive_node) then + let access_token = t.config.access_token in + let request_body = Request.Platform_agnostic.Dump_archive_data (id t) in + try + let%bind data = + match%map ci_http_request' ~access_token ~request_body with + | Ok (Response.Platform_agnostic.Archive_data_dump (node_id, logs)) + when String.equal node_id (id t) -> + logs + | _ -> + failwith "invalid archive dump data response" + in + [%log info] "Dumping archive data to file %s" data_file ; + Malleable_error.return + @@ Out_channel.with_file data_file ~f:(fun out_ch -> + Out_channel.output_string out_ch data ) + with Failure err -> Malleable_error.hard_error_string err + else + Malleable_error.hard_error_string + @@ sprintf "Node %s of type %s cannot dump archive data" (id t) + (Node_type.to_string t.node_type) let run_replayer ~logger (t : t) = [%log info] "Running replayer on archived data node: %s" t.node_id ; let access_token = t.config.access_token in let request_body = Request.Platform_agnostic.Run_replayer (id t) in - let%bind output = - match%map send_ci_http_request' ~access_token ~request_body with - | Ok (Response.Platform_agnostic.Replayer_run (node_id, output)) - when String.equal node_id (id t) -> - output - | _ -> - failwith "invalid run replayer response" - in - Malleable_error.return output + try + let%bind output = + match%map ci_http_request' ~access_token ~request_body with + | Ok (Response.Platform_agnostic.Replayer_run (node_id, output)) + when String.equal node_id (id t) -> + output + | _ -> + failwith "invalid run replayer response" + in + Malleable_error.return output + with Failure err -> Malleable_error.hard_error_string err - (* TODO: convert *) let dump_mina_logs ~logger (t : t) ~log_file = - let open Malleable_error.Let_syntax in [%log info] "Dumping logs from node: %s" t.node_id ; - let%map logs = get_logs_in_container t in - [%log info] "Dumping container log to file %s" log_file ; - Out_channel.with_file log_file ~f:(fun out_ch -> - Out_channel.output_string out_ch logs ) + let access_token = t.config.access_token in + let request_body = Request.Platform_agnostic.Dump_mina_logs (id t) in + try + let%bind logs = + match%map ci_http_request' ~access_token ~request_body with + | Ok (Response.Platform_agnostic.Mina_logs_dump (node_id, logs)) + when String.equal node_id (id t) -> + logs + | _ -> + failwith "invalid dump mina logs response" + in + [%log info] "Dumping logs to file %s" log_file ; + Malleable_error.return + @@ Out_channel.with_file log_file ~f:(fun out_ch -> + Out_channel.output_string out_ch logs ) + with Failure err -> Malleable_error.hard_error_string err - (* TODO: convert *) let dump_precomputed_blocks ~logger (t : t) = - let open Malleable_error.Let_syntax in [%log info] "Dumping precomputed blocks from logs for node: %s" t.node_id ; - let%bind logs = get_logs_in_container t in - (* kubectl logs may include non-log output, like "Using password from environment variable" *) - let log_lines = - String.split logs ~on:'\n' - |> List.filter ~f:(String.is_prefix ~prefix:"{\"timestamp\":") - in - let jsons = List.map log_lines ~f:Yojson.Safe.from_string in - let metadata_jsons = - List.map jsons ~f:(fun json -> - match json with - | `Assoc items -> ( - match List.Assoc.find items ~equal:String.equal "metadata" with - | Some md -> - md - | None -> - failwithf "Log line is missing metadata: %s" - (Yojson.Safe.to_string json) - () ) - | other -> - failwithf "Expected log line to be a JSON record, got: %s" - (Yojson.Safe.to_string other) - () ) - in - let state_hash_and_blocks = - List.fold metadata_jsons ~init:[] ~f:(fun acc json -> - match json with - | `Assoc items -> ( - match - List.Assoc.find items ~equal:String.equal "precomputed_block" - with - | Some block -> ( - match - List.Assoc.find items ~equal:String.equal "state_hash" - with - | Some state_hash -> - (state_hash, block) :: acc - | None -> - failwith - "Log metadata contains a precomputed block, but no \ - state hash" ) - | None -> - acc ) - | other -> - failwithf "Expected log line to be a JSON record, got: %s" - (Yojson.Safe.to_string other) - () ) - in - let%bind.Deferred () = - Deferred.List.iter state_hash_and_blocks - ~f:(fun (state_hash_json, block_json) -> - let double_quoted_state_hash = - Yojson.Safe.to_string state_hash_json - in - let state_hash = - String.sub double_quoted_state_hash ~pos:1 - ~len:(String.length double_quoted_state_hash - 2) - in - let block = Yojson.Safe.pretty_to_string block_json in - let filename = state_hash ^ ".json" in - match%map.Deferred Sys.file_exists filename with - | `Yes -> - [%log info] - "File already exists for precomputed block with state hash %s" - state_hash - | _ -> - [%log info] - "Dumping precomputed block with state hash %s to file %s" - state_hash filename ; - Out_channel.with_file filename ~f:(fun out_ch -> - Out_channel.output_string out_ch block ) ) + let access_token = t.config.access_token in + let request_body = + Request.Platform_agnostic.Dump_precomputed_blocks (id t) in - Malleable_error.return () + try + let%bind logs = + match%map ci_http_request' ~access_token ~request_body with + | Ok + (Response.Platform_agnostic.Precomputed_block_dump (node_id, blocks)) + when String.equal node_id (id t) -> + blocks + | _ -> + failwith "invalid dump precomputed blocks response" + in + let log_lines = + String.split logs ~on:'\n' + |> List.filter ~f:(String.is_prefix ~prefix:"{\"timestamp\":") + in + let jsons = List.map log_lines ~f:Yojson.Safe.from_string in + let metadata_jsons = + List.map jsons ~f:(fun json -> + match Yojson.Safe.Util.member "metadata" json with + | `Null -> + failwithf "Log line is missing metadata: %s" + (Yojson.Safe.to_string json) + () + | md -> + md ) + in + let state_hash_and_blocks = + List.fold metadata_jsons ~init:[] ~f:(fun acc json -> + match Yojson.Safe.Util.member "precomputed_block" json with + | `Null -> + acc + | `Assoc _ as block -> ( + match Yojson.Safe.Util.member "state_hash" json with + | `String _ as state_hash -> + (state_hash, block) :: acc + | _ -> + failwith + "Log metadata contains a precomputed block, but no state \ + hash" ) + | other -> + failwithf "Expected log line to be a JSON record, got: %s" + (Yojson.Safe.to_string other) + () ) + in + let open Deferred.Let_syntax in + let%bind () = + Deferred.List.iter state_hash_and_blocks + ~f:(fun (state_hash_json, block_json) -> + let double_quoted_state_hash = + Yojson.Safe.to_string state_hash_json + in + let state_hash = + String.sub double_quoted_state_hash ~pos:1 + ~len:(String.length double_quoted_state_hash - 2) + in + let block = Yojson.Safe.pretty_to_string block_json in + let filename = state_hash ^ ".json" in + match%map Sys.file_exists filename with + | `Yes -> + [%log info] + "File already exists for precomputed block with state hash %s" + state_hash + | _ -> + [%log info] + "Dumping precomputed block with state hash %s to file %s" + state_hash filename ; + Out_channel.with_file filename ~f:(fun out_ch -> + Out_channel.output_string out_ch block ) ) + in + Malleable_error.return () + with Failure err -> Malleable_error.hard_error_string err let get_metrics ~logger t = let open Deferred.Or_error.Let_syntax in @@ -1191,15 +1191,19 @@ module Node = struct } end +(* TODO: remove *) module Workload_to_deploy = struct type t = { workload_id : string; pod_info : string } + (* TODO: remove *) + let base_kube_args { cluster; namespace; _ } = + [ "--cluster"; cluster; "--namespace"; namespace ] + let construct_workload workload_id pod_info : t = { workload_id; pod_info } let[@warning "-27"] cons_pod_info ?network_keypair ?(has_archive_container = false) primary_container_id = "" - (* { network_keypair; has_archive_container; primary_container_id } *) let get_nodes_from_workload t ~config = let%bind cwd = Unix.getcwd () in @@ -1227,19 +1231,24 @@ module Workload_to_deploy = struct |> String.concat ~sep:"TODO:" in let node_type = Node_type.Seed_node in + (* TODO: get from ci runner *) let network_keypair = None in - { Node.app_id; node_id; node_type; config; network_keypair } + (* TODO: get from ci runner *) + let password = "password" in + (* TODO: get from ci runner *) + { Node.app_id; node_id; node_type; config; network_keypair; password } end type t = { constants : Test_config.constants + ; testnet_log_filter : string + ; genesis_keypairs : Network_keypair.t Core.String.Map.t + (* below values are given by CI *) ; seeds : Node.t Core.String.Map.t ; block_producers : Node.t Core.String.Map.t ; snark_coordinators : Node.t Core.String.Map.t ; snark_workers : Node.t Core.String.Map.t ; archive_nodes : Node.t Core.String.Map.t - ; testnet_log_filter : string - ; genesis_keypairs : Network_keypair.t Core.String.Map.t ; network_id : Network_id.t } @@ -1271,8 +1280,7 @@ let all_nodes { seeds; block_producers; snark_coordinators; archive_nodes; _ } = ] |> Core.String.Map.of_alist_exn -(* all_pods returns everything in the network. remember that snark_workers will never initialize and will never sync, and aren't supposed to *) -(* TODO: snark workers and snark coordinators have the same key name, but different workload ids*) +(* TODO: this info is returned from CI *) let all_pods t = List.concat [ Core.String.Map.to_alist t.seeds @@ -1299,7 +1307,7 @@ let all_node_id t = let pods = all_pods t |> Core.Map.to_alist in List.fold pods ~init:[] ~f:(fun acc (_, node) -> List.cons node.node_id acc) -(* TODO: what to do with this? *) +(* TODO: remove *) let initialize_infra ~logger network = let open Malleable_error.Let_syntax in let poll_interval = Time.Span.of_sec 15.0 in diff --git a/src/lib/integration_test_abstract_engine/cli_inputs.ml b/src/lib/integration_test_abstract_engine/cli_inputs.ml index cc66fdb8f79..5135757dcac 100644 --- a/src/lib/integration_test_abstract_engine/cli_inputs.ml +++ b/src/lib/integration_test_abstract_engine/cli_inputs.ml @@ -17,4 +17,5 @@ let term = [ "mina-automation-location" ] ~env ~docv:"MINA_AUTOMATION_LOCATION" ~doc) in - mina_automation_location + let lift mina_automation_location = { mina_automation_location } in + Term.(const lift $ mina_automation_location) diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index 4ff32738248..57f39e1fe44 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -21,8 +21,6 @@ type best_chain_block = ; global_slot_since_hard_fork : Mina_numbers.Global_slot_since_hard_fork.t } -(* TODO: malleable error -> or error *) - module Engine = struct module type Network_config_intf = sig module Cli_inputs : sig From 29e1ce54ff332e3cbc69f6d488558701c04f9737 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sun, 30 Jul 2023 04:31:28 +0000 Subject: [PATCH 14/90] (test) Add config path ref --- src/lib/integration_test_lib/intf.ml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index 57f39e1fe44..5982f2e7008 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -104,6 +104,8 @@ module Engine = struct val initialize_infra : logger:Logger.t -> t -> unit Malleable_error.t val id : t -> string + + val config_path : string ref end module type Network_manager_intf = sig @@ -339,16 +341,16 @@ module Test = struct type dsl + val test_name : string + val config : Test_config.t val run : network -> dsl -> unit Malleable_error.t end - (* NB: until the DSL is actually implemented, a test just takes in the engine - * implementation directly. *) module type Functor_intf = functor (Inputs : Inputs_intf) -> S - with type network = Inputs.Engine.Network.t - and type node = Inputs.Engine.Network.Node.t - and type dsl = Inputs.Dsl.t + with type network := Inputs.Engine.Network.t + and type node := Inputs.Engine.Network.Node.t + and type dsl := Inputs.Dsl.t end From 58dee3401791899a06437935e50c73bb1efb11d5 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sun, 30 Jul 2023 04:34:30 +0000 Subject: [PATCH 15/90] (test) Update tests and remove intf files --- src/app/test_executive/block_production_priority.ml | 12 ++---------- src/app/test_executive/block_production_priority.mli | 1 - .../block_production_timed_accounts_test.mli.todo | 1 - src/app/test_executive/block_reward_test.ml | 10 ++-------- src/app/test_executive/block_reward_test.mli | 1 - src/app/test_executive/chain_reliability_test.ml | 10 ++-------- src/app/test_executive/chain_reliability_test.mli | 1 - src/app/test_executive/gossip_consistency.ml | 10 ++-------- src/app/test_executive/gossip_consistency.mli | 1 - src/app/test_executive/medium_bootstrap.ml | 8 +------- src/app/test_executive/medium_bootstrap.mli | 1 - src/app/test_executive/payments_test.ml | 10 ++-------- src/app/test_executive/payments_test.mli | 1 - src/app/test_executive/peers_reliability_test.ml | 10 ++-------- src/app/test_executive/peers_reliability_test.mli | 1 - src/app/test_executive/snarkyjs.ml | 9 ++------- src/app/test_executive/verification_key_update.ml | 9 ++------- src/app/test_executive/zkapps.ml | 9 ++------- src/app/test_executive/zkapps_nonce_test.ml | 9 ++------- src/app/test_executive/zkapps_timing.ml | 9 ++------- 20 files changed, 23 insertions(+), 100 deletions(-) delete mode 100644 src/app/test_executive/block_production_priority.mli delete mode 100644 src/app/test_executive/block_production_timed_accounts_test.mli.todo delete mode 100644 src/app/test_executive/block_reward_test.mli delete mode 100644 src/app/test_executive/chain_reliability_test.mli delete mode 100644 src/app/test_executive/gossip_consistency.mli delete mode 100644 src/app/test_executive/medium_bootstrap.mli delete mode 100644 src/app/test_executive/payments_test.mli delete mode 100644 src/app/test_executive/peers_reliability_test.mli diff --git a/src/app/test_executive/block_production_priority.ml b/src/app/test_executive/block_production_priority.ml index e46ec8d474f..47353bc09ad 100644 --- a/src/app/test_executive/block_production_priority.ml +++ b/src/app/test_executive/block_production_priority.ml @@ -8,22 +8,14 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct open Test_common.Make (Inputs) - (* TODO: find a way to avoid this type alias (first class module signatures restrictions make this tricky) *) - type network = Network.t - - type node = Network.Node.t - - type dsl = Dsl.t - let num_extra_keys = 1000 - (* let num_sender_nodes = 4 *) + let test_name = "block-prod-prio" let config = let open Test_config in { default with - requires_graphql = true - ; genesis_ledger = + genesis_ledger = [ { Test_Account.account_name = "receiver-key" ; balance = "9999999" ; timing = Untimed diff --git a/src/app/test_executive/block_production_priority.mli b/src/app/test_executive/block_production_priority.mli deleted file mode 100644 index 8664ff022c1..00000000000 --- a/src/app/test_executive/block_production_priority.mli +++ /dev/null @@ -1 +0,0 @@ -module Make : Integration_test_lib.Intf.Test.Functor_intf diff --git a/src/app/test_executive/block_production_timed_accounts_test.mli.todo b/src/app/test_executive/block_production_timed_accounts_test.mli.todo deleted file mode 100644 index 28287c1dfa7..00000000000 --- a/src/app/test_executive/block_production_timed_accounts_test.mli.todo +++ /dev/null @@ -1 +0,0 @@ -module Make : Integration_test_lib.Test_functor_intf diff --git a/src/app/test_executive/block_reward_test.ml b/src/app/test_executive/block_reward_test.ml index cf30894453e..63b11d22fc7 100644 --- a/src/app/test_executive/block_reward_test.ml +++ b/src/app/test_executive/block_reward_test.ml @@ -9,18 +9,12 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct open Test_common.Make (Inputs) - (* TODO: find a way to avoid this type alias (first class module signatures restrictions make this tricky) *) - type network = Network.t - - type node = Network.Node.t - - type dsl = Dsl.t + let test_name = "block-reward" let config = let open Test_config in { default with - requires_graphql = true - ; genesis_ledger = + genesis_ledger = [ { account_name = "node-key"; balance = "1000"; timing = Untimed } ] ; block_producers = [ { node_name = "node"; account_name = "node-key" } ] } diff --git a/src/app/test_executive/block_reward_test.mli b/src/app/test_executive/block_reward_test.mli deleted file mode 100644 index 8664ff022c1..00000000000 --- a/src/app/test_executive/block_reward_test.mli +++ /dev/null @@ -1 +0,0 @@ -module Make : Integration_test_lib.Intf.Test.Functor_intf diff --git a/src/app/test_executive/chain_reliability_test.ml b/src/app/test_executive/chain_reliability_test.ml index 435c4795b7f..920a773b17d 100644 --- a/src/app/test_executive/chain_reliability_test.ml +++ b/src/app/test_executive/chain_reliability_test.ml @@ -8,18 +8,12 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct open Test_common.Make (Inputs) - (* TODO: find a way to avoid this type alias (first class module signatures restrictions make this tricky) *) - type network = Network.t - - type node = Network.Node.t - - type dsl = Dsl.t + let test_name = "chain-reliability" let config = let open Test_config in { default with - requires_graphql = true - ; genesis_ledger = + genesis_ledger = [ { account_name = "node-a-key"; balance = "1000"; timing = Untimed } ; { account_name = "node-b-key"; balance = "1000"; timing = Untimed } ; { account_name = "node-c-key"; balance = "0"; timing = Untimed } diff --git a/src/app/test_executive/chain_reliability_test.mli b/src/app/test_executive/chain_reliability_test.mli deleted file mode 100644 index 8664ff022c1..00000000000 --- a/src/app/test_executive/chain_reliability_test.mli +++ /dev/null @@ -1 +0,0 @@ -module Make : Integration_test_lib.Intf.Test.Functor_intf diff --git a/src/app/test_executive/gossip_consistency.ml b/src/app/test_executive/gossip_consistency.ml index 03b309e9d90..4229ca320be 100644 --- a/src/app/test_executive/gossip_consistency.ml +++ b/src/app/test_executive/gossip_consistency.ml @@ -8,18 +8,12 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct open Test_common.Make (Inputs) - (* TODO: find a way to avoid this type alias (first class module signatures restrictions make this tricky) *) - type network = Network.t - - type node = Network.Node.t - - type dsl = Dsl.t + let test_name = "gossip-consis" let config = let open Test_config in { default with - requires_graphql = true - ; genesis_ledger = + genesis_ledger = [ { account_name = "node-a-key"; balance = "1000"; timing = Untimed } ; { account_name = "node-b-key"; balance = "1000"; timing = Untimed } ] diff --git a/src/app/test_executive/gossip_consistency.mli b/src/app/test_executive/gossip_consistency.mli deleted file mode 100644 index 8664ff022c1..00000000000 --- a/src/app/test_executive/gossip_consistency.mli +++ /dev/null @@ -1 +0,0 @@ -module Make : Integration_test_lib.Intf.Test.Functor_intf diff --git a/src/app/test_executive/medium_bootstrap.ml b/src/app/test_executive/medium_bootstrap.ml index 92fce3515b3..b13c0728fb8 100644 --- a/src/app/test_executive/medium_bootstrap.ml +++ b/src/app/test_executive/medium_bootstrap.ml @@ -9,18 +9,12 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct open Test_common.Make (Inputs) - (* TODO: find a way to avoid this type alias (first class module signatures restrictions make this tricky) *) - type network = Network.t - - type node = Network.Node.t - - type dsl = Dsl.t + let test_name = "medium-bootstrap" let config = let open Test_config in { default with k = 2 - ; requires_graphql = true ; genesis_ledger = [ { account_name = "node-a-key"; balance = "1000"; timing = Untimed } ; { account_name = "node-b-key"; balance = "1000"; timing = Untimed } diff --git a/src/app/test_executive/medium_bootstrap.mli b/src/app/test_executive/medium_bootstrap.mli deleted file mode 100644 index 8664ff022c1..00000000000 --- a/src/app/test_executive/medium_bootstrap.mli +++ /dev/null @@ -1 +0,0 @@ -module Make : Integration_test_lib.Intf.Test.Functor_intf diff --git a/src/app/test_executive/payments_test.ml b/src/app/test_executive/payments_test.ml index 56cfde89101..aad1807f0f9 100644 --- a/src/app/test_executive/payments_test.ml +++ b/src/app/test_executive/payments_test.ml @@ -10,20 +10,14 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct open Test_common.Make (Inputs) - (* TODO: find a way to avoid this type alias (first class module signatures restrictions make this tricky) *) - type network = Network.t - - type node = Network.Node.t - - type dsl = Dsl.t + let test_name = "payments" (* TODO: refactor all currency values to decimal represenation *) (* TODO: test account creation fee *) let config = let open Test_config in { default with - requires_graphql = true - ; genesis_ledger = + genesis_ledger = [ { account_name = "untimed-node-a-key" ; balance = "400000" ; timing = Untimed (* 400_000_000_000_000 *) diff --git a/src/app/test_executive/payments_test.mli b/src/app/test_executive/payments_test.mli deleted file mode 100644 index 8664ff022c1..00000000000 --- a/src/app/test_executive/payments_test.mli +++ /dev/null @@ -1 +0,0 @@ -module Make : Integration_test_lib.Intf.Test.Functor_intf diff --git a/src/app/test_executive/peers_reliability_test.ml b/src/app/test_executive/peers_reliability_test.ml index 253358f3821..98196416874 100644 --- a/src/app/test_executive/peers_reliability_test.ml +++ b/src/app/test_executive/peers_reliability_test.ml @@ -9,18 +9,12 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct open Test_common.Make (Inputs) - (* TODO: find a way to avoid this type alias (first class module signatures restrictions make this tricky) *) - type network = Network.t - - type node = Network.Node.t - - type dsl = Dsl.t + let test_name = "peer-reliability" let config = let open Test_config in { default with - requires_graphql = true - ; genesis_ledger = + genesis_ledger = [ { account_name = "node-a-key"; balance = "700000"; timing = Untimed } ; { account_name = "node-b-key"; balance = "700000"; timing = Untimed } ; { account_name = "node-c-key"; balance = "800000"; timing = Untimed } diff --git a/src/app/test_executive/peers_reliability_test.mli b/src/app/test_executive/peers_reliability_test.mli deleted file mode 100644 index 8664ff022c1..00000000000 --- a/src/app/test_executive/peers_reliability_test.mli +++ /dev/null @@ -1 +0,0 @@ -module Make : Integration_test_lib.Intf.Test.Functor_intf diff --git a/src/app/test_executive/snarkyjs.ml b/src/app/test_executive/snarkyjs.ml index 70730d4c16d..9772b024c5c 100644 --- a/src/app/test_executive/snarkyjs.ml +++ b/src/app/test_executive/snarkyjs.ml @@ -9,17 +9,12 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct open Test_common.Make (Inputs) - type network = Network.t - - type node = Network.Node.t - - type dsl = Dsl.t + let test_name = "snarkyjs" let config = let open Test_config in { default with - requires_graphql = true - ; genesis_ledger = + genesis_ledger = [ { account_name = "node-key"; balance = "8000000"; timing = Untimed } ; { account_name = "extra-key"; balance = "10"; timing = Untimed } ] diff --git a/src/app/test_executive/verification_key_update.ml b/src/app/test_executive/verification_key_update.ml index 7822816ca13..e741aa35396 100644 --- a/src/app/test_executive/verification_key_update.ml +++ b/src/app/test_executive/verification_key_update.ml @@ -61,17 +61,12 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct open Test_common.Make (Inputs) - type network = Network.t - - type node = Network.Node.t - - type dsl = Dsl.t + let test_name = "verification-key" let config = let open Test_config in { default with - requires_graphql = true - ; genesis_ledger = + genesis_ledger = [ { account_name = "whale1-key" ; balance = "9000000000" ; timing = Untimed diff --git a/src/app/test_executive/zkapps.ml b/src/app/test_executive/zkapps.ml index 624dca72828..32f816f1243 100644 --- a/src/app/test_executive/zkapps.ml +++ b/src/app/test_executive/zkapps.ml @@ -10,17 +10,12 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct open Test_common.Make (Inputs) - type network = Network.t - - type node = Network.Node.t - - type dsl = Dsl.t + let test_name = "zkapps" let config = let open Test_config in { default with - requires_graphql = true - ; genesis_ledger = + genesis_ledger = [ { account_name = "node-a-key" ; balance = "8000000000" ; timing = Untimed diff --git a/src/app/test_executive/zkapps_nonce_test.ml b/src/app/test_executive/zkapps_nonce_test.ml index dad202c8f29..7ff563285ca 100644 --- a/src/app/test_executive/zkapps_nonce_test.ml +++ b/src/app/test_executive/zkapps_nonce_test.ml @@ -10,17 +10,12 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct open Test_common.Make (Inputs) - type network = Network.t - - type node = Network.Node.t - - type dsl = Dsl.t + let test_name = "zkapps-nonce" let config = let open Test_config in { default with - requires_graphql = true - ; genesis_ledger = + genesis_ledger = [ { account_name = "node-a-key" ; balance = "8000000000" ; timing = Untimed diff --git a/src/app/test_executive/zkapps_timing.ml b/src/app/test_executive/zkapps_timing.ml index bd65683c726..c763117e6df 100644 --- a/src/app/test_executive/zkapps_timing.ml +++ b/src/app/test_executive/zkapps_timing.ml @@ -9,17 +9,12 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct open Test_common.Make (Inputs) - type network = Network.t - - type node = Network.Node.t - - type dsl = Dsl.t + let test_name = "zkapps-timing" let config = let open Test_config in { default with - requires_graphql = true - ; genesis_ledger = + genesis_ledger = [ { account_name = "node-a-key" ; balance = "8000000000" ; timing = Untimed From e49bee0615df848bda600837e0d101659da4748d Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sun, 30 Jul 2023 04:36:15 +0000 Subject: [PATCH 16/90] (test exec) Clean up and handle config path --- src/app/test_executive/test_executive.ml | 97 ++++++++++++++++-------- 1 file changed, 66 insertions(+), 31 deletions(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index 2e9e283a605..5b8892c0fb5 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -4,7 +4,7 @@ open Cmdliner open Pipe_lib open Integration_test_lib -type test = string * (module Intf.Test.Functor_intf) +type test = (module Intf.Test.Functor_intf) type engine = string * (module Intf.Engine.S) @@ -30,6 +30,7 @@ type inputs = ; mina_image : string ; archive_image : string option ; debug : bool + ; config_path : string } let validate_inputs ~logger inputs (test_config : Test_config.t) : @@ -44,10 +45,13 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : "This test uses archive nodes. archive-image argument cannot be absent \ for this test" ; exit 1 ) - else Deferred.return () + else Deferred.unit let engines : engine list = - [ ("cloud", (module Integration_test_abstract_engine : Intf.Engine.S)) ] + let abstract_engine = + (module Integration_test_abstract_engine : Intf.Engine.S) + in + [ ("cloud", abstract_engine); ("local", abstract_engine) ] let tests : test list = [ ( "peers-reliability" @@ -196,8 +200,6 @@ let report_test_errors ~log_error_set ~internal_error_set = let%bind () = Writer.(flushed (Lazy.force stderr)) in return exit_code -(* TODO: refactor cleanup system (smells like a monad for composing linear resources would help a lot) *) - let dispatch_cleanup ~logger ~pause_cleanup_func ~network_cleanup_func ~log_engine_cleanup_func ~lift_accumulated_errors_func ~net_manager_ref ~log_engine_ref ~network_state_writer_ref ~cleanup_deferred_ref ~exit_reason @@ -240,19 +242,22 @@ let dispatch_cleanup ~logger ~pause_cleanup_func ~network_cleanup_func cleanup_deferred_ref := Some deferred ; deferred +let test_name (test : test) + (inputs : (module Integration_test_lib.Intf.Test.Inputs_intf)) = + let (module Inputs) = inputs in + let module Test = (val test) (Inputs) in + Test.test_name + let main inputs = let (Test_inputs_with_cli_inputs ((module Test_inputs), cli_inputs)) = inputs.test_inputs in let open Test_inputs in - let test_name, (module Test) = inputs.test in - let (module T) = - (module Test (Test_inputs) : Intf.Test.S - with type network = Engine.Network.t - and type node = Engine.Network.Node.t - and type dsl = Dsl.t ) - in + let (module Test) = inputs.test in + let test_name = test_name inputs.test (module Test_inputs) in + let module T = Test (Test_inputs) in let logger = Logger.create () in + (* TODO: provide as json file *) let images = { Test_config.Container_images.mina = inputs.mina_image ; archive_node = @@ -291,7 +296,7 @@ let main inputs = ~lift_accumulated_errors_func ~net_manager_ref ~log_engine_ref ~network_state_writer_ref ~cleanup_deferred_ref in - (* run test while gracefully recovering handling exceptions and interrupts *) + (* run test while gracefully recovering, handling exceptions, and interrupts *) [%log trace] "attaching signal handler" ; Signal.handle Signal.terminating ~f:(fun signal -> [%log info] "handling signal %s" (Signal.to_string signal) ; @@ -349,7 +354,7 @@ let main inputs = in [%log trace] "initializing network abstraction" ; let%bind () = Engine.Network.initialize_infra ~logger network in - [%log info] "Starting the daemons within the pods" ; + [%log info] "starting the daemons within the pods" ; let start_print (node : Engine.Network.Node.t) = let open Malleable_error.Let_syntax in [%log info] "starting %s ..." (Engine.Network.Node.id node) ; @@ -384,7 +389,7 @@ let main inputs = Dsl.wait_for dsl (Dsl.Wait_condition.nodes_to_initialize seed_nodes) in let%bind () = Malleable_error.List.iter non_seed_pods ~f:start_print in - [%log info] "Daemons started" ; + [%log info] "daemons started" ; [%log trace] "executing test" ; T.run network dsl ) in @@ -407,14 +412,21 @@ let start inputs = never_returns (Async.Scheduler.go_main ~main:(fun () -> don't_wait_for (main inputs)) ()) -let test_arg = - (* we nest the tests in a redundant index so that we still get the name back after cmdliner evaluates the argument *) +let test_arg inputs = let indexed_tests = - List.map tests ~f:(fun (name, test) -> (name, (name, test))) + List.map tests ~f:(fun test -> (test_name test inputs, test)) in let doc = "The name of the test to execute." in Arg.(required & pos 0 (some (enum indexed_tests)) None & info [] ~doc) +let config_path_arg = + let doc = "Path to the CI config file." in + let env = Arg.env_var "MINA_CI_CONFIG_PATH" ~doc in + Arg.( + required + & opt (some non_dir_file) None + & info [ "config-path"; "config" ] ~env ~docv:"MINA_CI_CONFIG_PATH" ~doc) + let mina_image_arg = let doc = "Identifier of the Mina docker image to test." in let env = Arg.env_var "MINA_IMAGE" ~doc in @@ -423,13 +435,21 @@ let mina_image_arg = & opt (some string) None & info [ "mina-image" ] ~env ~docv:"MINA_IMAGE" ~doc) +let images_arg = + let doc = "Identifier of the archive node docker image to test." in + let env = Arg.env_var "ARCHIVE_IMAGE" ~doc in + Arg.( + value + & opt (some string) None + & info [ "archive-image" ] ~env ~docv:"ARCHIVE_IMAGE" ~doc) + let archive_image_arg = let doc = "Identifier of the archive node docker image to test." in let env = Arg.env_var "ARCHIVE_IMAGE" ~doc in Arg.( value - ( opt (some string) None - & info [ "archive-image" ] ~env ~docv:"ARCHIVE_IMAGE" ~doc )) + & opt (some string) None + & info [ "archive-image" ] ~env ~docv:"ARCHIVE_IMAGE" ~doc) let debug_arg = let doc = @@ -440,28 +460,43 @@ let debug_arg = let help_term = Term.(ret @@ const (`Help (`Plain, None))) +let info engine_name = + let doc = + match engine_name with + | "cloud" -> + "Run mina integration tests on a remote cloud provider." + | "local" -> + "Run mina integration tests locally with Minimina." + | _ -> + assert false + in + Term.info engine_name ~doc ~exits:Term.default_exits + let engine_cmd ((engine_name, (module Engine)) : engine) = - let info = - let doc = "Run mina integration test(s) on remote cloud provider." in - Term.info engine_name ~doc ~exits:Term.default_exits + let info = info engine_name in + let module Inputs = Make_test_inputs (Engine) () in + let set_config path = + Engine.Network.config_path := path ; + path in let test_inputs_with_cli_inputs_arg = let wrap_cli_inputs cli_inputs = - Test_inputs_with_cli_inputs - ((module Make_test_inputs (Engine) ()), cli_inputs) + Test_inputs_with_cli_inputs ((module Inputs), cli_inputs) in Term.(const wrap_cli_inputs $ Engine.Network_config.Cli_inputs.term) in let inputs_term = - let cons_inputs test_inputs test mina_image archive_image debug = - { test_inputs; test; mina_image; archive_image; debug } + let cons_inputs test_inputs test archive_image debug mina_image config_path + = + { test_inputs; test; mina_image; archive_image; debug; config_path } in Term.( - const cons_inputs $ test_inputs_with_cli_inputs_arg $ test_arg - $ mina_image_arg $ archive_image_arg $ debug_arg) + const cons_inputs $ test_inputs_with_cli_inputs_arg + $ test_arg (module Inputs) + $ archive_image_arg $ debug_arg $ mina_image_arg + $ (const set_config $ config_path_arg)) in - let term = Term.(const start $ inputs_term) in - (term, info) + (Term.(const start $ inputs_term), info) let help_cmd = let doc = "Print out test executive documentation." in From f749d48783b907e7fed113011124a6f442bace9b Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sun, 30 Jul 2023 04:37:55 +0000 Subject: [PATCH 17/90] (test exec) Replace http requests with config commands --- .../abstract_network.ml | 423 ++++++++---------- 1 file changed, 198 insertions(+), 225 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index b430770712d..815a883ad85 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -12,17 +12,17 @@ type config = ; cluster : string ; namespace : string ; graphql_enabled : bool - ; access_token : Access_token.t ; network_id : Network_id.t ; ingress_uri : string ; current_commit_sha : string } +(* [config_path] is instantiated when command line args are parsed *) +let config_path = ref "" + module Node = struct - (* TODO: remove app_id, add ingress_uri *) type t = - { app_id : string - ; node_id : string + { node_id : string ; network_keypair : Network_keypair.t option ; config : config ; node_type : Node_type.t @@ -44,40 +44,41 @@ module Node = struct (base_kube_args config @ [ "logs"; "-c"; container_id; node_id ]) let start ?commit_sha ~fresh_state t : unit Malleable_error.t = - let commit_sha = - Option.value commit_sha ~default:t.config.current_commit_sha - in - let access_token = t.config.access_token in - let request_body = - Request.Platform_agnostic.Start_node - { node_id = id t; fresh_state; commit_sha } - in - let%bind () = - match%map ci_http_request' ~access_token ~request_body with - | Ok (Response.Platform_agnostic.Node_started node_id) - when String.equal node_id (id t) -> - () - | _ -> - failwith "invalid node started response" - in - Malleable_error.return () + try + let commit_sha = + Option.value commit_sha ~default:t.config.current_commit_sha + in + let args = + [ ("node_id", `String (id t)) + ; ("fresh_state", `Bool fresh_state) + ; ("commit_sha", `String commit_sha) + ] + in + let%bind () = + match%map run_command ~config:!config_path ~args "start_node" with + | Ok (Response.Node_started _) -> + () + | _ -> + failwith "invalid node started response" + in + Malleable_error.return () + with Failure err -> Malleable_error.hard_error_string err let stop t = - let access_token = t.config.access_token in - let request_body = Request.Platform_agnostic.Stop_node (id t) in - let%bind () = - match%map ci_http_request' ~access_token ~request_body with - | Ok (Response.Platform_agnostic.Node_stopped node_id) - when String.equal node_id (id t) -> - () - | _ -> - failwith "invalid node stopped response" - in - Malleable_error.return () + try + let args = [ ("node_id", `String (id t)) ] in + let%bind () = + match%map run_command ~config:!config_path ~args "stop_node" with + | Ok (Response.Node_stopped _) -> + () + | _ -> + failwith "invalid node stopped response" + in + Malleable_error.return () + with Failure err -> Malleable_error.hard_error_string err let logger_metadata node = [ ("namespace", `String node.config.namespace) - ; ("app_id", `String node.app_id) ; ("node_id", `String node.node_id) ] @@ -89,7 +90,7 @@ module Node = struct let host = sprintf "%s.%s" node.config.testnet_name node.config.ingress_uri in - let path = sprintf "/%s/graphql" node.app_id in + let path = sprintf "/%s/graphql" node.node_id in Uri.make ~scheme:"http" ~host ~path ~port:80 () module Client = Graphql_lib.Client.Make (struct @@ -105,104 +106,104 @@ module Node = struct module Unlock_account = [%graphql ({| - mutation ($password: String!, $public_key: PublicKey!) @encoders(module: "Encoders"){ - unlockAccount(input: {password: $password, publicKey: $public_key }) { - account { - public_key: publicKey + mutation ($password: String!, $public_key: PublicKey!) @encoders(module: "Encoders"){ + unlockAccount(input: {password: $password, publicKey: $public_key }) { + account { + public_key: publicKey + } } } - } - |} + |} [@encoders Encoders] )] module Send_test_payments = [%graphql {| - mutation ($senders: [PrivateKey!]!, - $receiver: PublicKey!, - $amount: UInt64!, - $fee: UInt64!, - $repeat_count: UInt32!, - $repeat_delay_ms: UInt32!) @encoders(module: "Encoders"){ - sendTestPayments( - senders: $senders, receiver: $receiver, amount: $amount, fee: $fee, - repeat_count: $repeat_count, - repeat_delay_ms: $repeat_delay_ms) - } - |}] + mutation ($senders: [PrivateKey!]!, + $receiver: PublicKey!, + $amount: UInt64!, + $fee: UInt64!, + $repeat_count: UInt32!, + $repeat_delay_ms: UInt32!) @encoders(module: "Encoders"){ + sendTestPayments( + senders: $senders, receiver: $receiver, amount: $amount, fee: $fee, + repeat_count: $repeat_count, + repeat_delay_ms: $repeat_delay_ms) + } + |}] module Send_payment = [%graphql {| - mutation ($input: SendPaymentInput!)@encoders(module: "Encoders"){ - sendPayment(input: $input){ - payment { - id - nonce - hash + mutation ($input: SendPaymentInput!)@encoders(module: "Encoders"){ + sendPayment(input: $input){ + payment { + id + nonce + hash + } } - } - } - |}] + } + |}] module Send_payment_with_raw_sig = [%graphql {| - mutation ( - $input:SendPaymentInput!, - $rawSignature: String! - )@encoders(module: "Encoders") - { - sendPayment( - input:$input, - signature: {rawSignature: $rawSignature} - ) + mutation ( + $input:SendPaymentInput!, + $rawSignature: String! + )@encoders(module: "Encoders") { - payment { - id - nonce - hash + sendPayment( + input:$input, + signature: {rawSignature: $rawSignature} + ) + { + payment { + id + nonce + hash + } } } - } - |}] + |}] module Send_delegation = [%graphql {| - mutation ($input: SendDelegationInput!) @encoders(module: "Encoders"){ - sendDelegation(input:$input){ - delegation { - id - nonce - hash + mutation ($input: SendDelegationInput!) @encoders(module: "Encoders"){ + sendDelegation(input:$input){ + delegation { + id + nonce + hash + } } - } - } - |}] + } + |}] module Set_snark_worker = [%graphql {| - mutation ($input: SetSnarkWorkerInput! ) @encoders(module: "Encoders"){ - setSnarkWorker(input:$input){ - lastSnarkWorker - } - } - |}] + mutation ($input: SetSnarkWorkerInput! ) @encoders(module: "Encoders"){ + setSnarkWorker(input:$input){ + lastSnarkWorker + } + } + |}] module Get_account_data = [%graphql {| - query ($public_key: PublicKey!) @encoders(module: "Encoders"){ - account(publicKey: $public_key) { - nonce - balance { - total @ppxCustom(module: "Scalars.Balance") + query ($public_key: PublicKey!) @encoders(module: "Encoders"){ + account(publicKey: $public_key) { + nonce + balance { + total @ppxCustom(module: "Scalars.Balance") + } } } - } - |}] + |}] module Send_test_zkapp = Generated_graphql_queries.Send_test_zkapp module Pooled_zkapp_commands = @@ -211,112 +212,112 @@ module Node = struct module Query_peer_id = [%graphql {| - query { - daemonStatus { - addrsAndPorts { - peer { - peerId + query { + daemonStatus { + addrsAndPorts { + peer { + peerId + } } - } - peers { peerId } + peers { peerId } + } } - } - |}] + |}] module Best_chain = [%graphql {| - query ($max_length: Int) @encoders(module: "Encoders"){ - bestChain (maxLength: $max_length) { - stateHash @ppxCustom(module: "Graphql_lib.Scalars.String_json") - commandTransactionCount - creatorAccount { - publicKey @ppxCustom(module: "Graphql_lib.Scalars.JSON") + query ($max_length: Int) @encoders(module: "Encoders"){ + bestChain (maxLength: $max_length) { + stateHash @ppxCustom(module: "Graphql_lib.Scalars.String_json") + commandTransactionCount + creatorAccount { + publicKey @ppxCustom(module: "Graphql_lib.Scalars.JSON") + } } } - } - |}] + |}] module Query_metrics = [%graphql {| - query { - daemonStatus { - metrics { - blockProductionDelay - transactionPoolDiffReceived - transactionPoolDiffBroadcasted - transactionsAddedToPool - transactionPoolSize + query { + daemonStatus { + metrics { + blockProductionDelay + transactionPoolDiffReceived + transactionPoolDiffBroadcasted + transactionsAddedToPool + transactionPoolSize + } } } - } - |}] + |}] module StartFilteredLog = [%graphql {| - mutation ($filter: [String!]!) @encoders(module: "Encoders"){ - startFilteredLog(filter: $filter) - } - |}] + mutation ($filter: [String!]!) @encoders(module: "Encoders"){ + startFilteredLog(filter: $filter) + } + |}] module GetFilteredLogEntries = [%graphql {| - query ($offset: Int!) @encoders(module: "Encoders"){ - getFilteredLogEntries(offset: $offset) { - logMessages, - isCapturing, - } + query ($offset: Int!) @encoders(module: "Encoders"){ + getFilteredLogEntries(offset: $offset) { + logMessages, + isCapturing, + } - } - |}] + } + |}] module Account = [%graphql {| - query ($public_key: PublicKey!, $token: UInt64) { - account (publicKey : $public_key, token : $token) { - balance { liquid - locked - total + query ($public_key: PublicKey!, $token: UInt64) { + account (publicKey : $public_key, token : $token) { + balance { liquid + locked + total + } + delegate + nonce + permissions { editActionState + editState + incrementNonce + receive + send + access + setDelegate + setPermissions + setZkappUri + setTokenSymbol + setVerificationKey + setVotingFor + setTiming + } + actionState + zkappState + zkappUri + timing { cliffTime @ppxCustom(module: "Graphql_lib.Scalars.JSON") + cliffAmount + vestingPeriod @ppxCustom(module: "Graphql_lib.Scalars.JSON") + vestingIncrement + initialMinimumBalance } - delegate - nonce - permissions { editActionState - editState - incrementNonce - receive - send - access - setDelegate - setPermissions - setZkappUri - setTokenSymbol - setVerificationKey - setVotingFor - setTiming - } - actionState - zkappState - zkappUri - timing { cliffTime @ppxCustom(module: "Graphql_lib.Scalars.JSON") - cliffAmount - vestingPeriod @ppxCustom(module: "Graphql_lib.Scalars.JSON") - vestingIncrement - initialMinimumBalance - } - tokenId - tokenSymbol - verificationKey { verificationKey - hash - } - votingFor + tokenId + tokenSymbol + verificationKey { verificationKey + hash + } + votingFor + } } - } - |}] + |}] end (* this function will repeatedly attempt to connect to graphql port times before giving up *) @@ -368,7 +369,7 @@ module Node = struct [%log error] "GraphQL request \"$query\" to \"$uri\" returned an error: \ \"$error\" (this is a graphql error so not retrying)" - ~metadata:(metadata @ [ ("error", `String err_string) ]) ; + ~metadata:(("error", `String err_string) :: metadata) ; Deferred.Or_error.error_string err_string in let%bind () = after (Time.Span.of_sec initial_delay_sec) in @@ -998,7 +999,7 @@ module Node = struct [%log error] "Attempted to start structured log collection on $node, but it had \ already started" - ~metadata:[ ("node", `String t.app_id) ] ; + ~metadata:[ ("node", `String t.node_id) ] ; return (Ok ()) ) | Error e -> return (Error e) @@ -1021,13 +1022,13 @@ module Node = struct let dump_archive_data ~logger (t : t) ~data_file = if Node_type.(equal t.node_type Archive_node) then - let access_token = t.config.access_token in - let request_body = Request.Platform_agnostic.Dump_archive_data (id t) in + let args = [ ("node_id", `String (id t)) ] in try let%bind data = - match%map ci_http_request' ~access_token ~request_body with - | Ok (Response.Platform_agnostic.Archive_data_dump (node_id, logs)) - when String.equal node_id (id t) -> + match%map + run_command ~config:!config_path ~args "dump_archive_data" + with + | Ok (Response.Archive_data_dump (_, logs)) -> logs | _ -> failwith "invalid archive dump data response" @@ -1044,13 +1045,11 @@ module Node = struct let run_replayer ~logger (t : t) = [%log info] "Running replayer on archived data node: %s" t.node_id ; - let access_token = t.config.access_token in - let request_body = Request.Platform_agnostic.Run_replayer (id t) in + let args = [ ("node_id", `String (id t)) ] in try let%bind output = - match%map ci_http_request' ~access_token ~request_body with - | Ok (Response.Platform_agnostic.Replayer_run (node_id, output)) - when String.equal node_id (id t) -> + match%map run_command ~config:!config_path ~args "run_replayer" with + | Ok (Response.Replayer_run (_, output)) -> output | _ -> failwith "invalid run replayer response" @@ -1060,13 +1059,11 @@ module Node = struct let dump_mina_logs ~logger (t : t) ~log_file = [%log info] "Dumping logs from node: %s" t.node_id ; - let access_token = t.config.access_token in - let request_body = Request.Platform_agnostic.Dump_mina_logs (id t) in + let args = [ ("node_id", `String (id t)) ] in try let%bind logs = - match%map ci_http_request' ~access_token ~request_body with - | Ok (Response.Platform_agnostic.Mina_logs_dump (node_id, logs)) - when String.equal node_id (id t) -> + match%map run_command ~config:!config_path ~args "dump_mina_logs" with + | Ok (Response.Mina_logs_dump (_, logs)) -> logs | _ -> failwith "invalid dump mina logs response" @@ -1079,16 +1076,13 @@ module Node = struct let dump_precomputed_blocks ~logger (t : t) = [%log info] "Dumping precomputed blocks from logs for node: %s" t.node_id ; - let access_token = t.config.access_token in - let request_body = - Request.Platform_agnostic.Dump_precomputed_blocks (id t) - in + let args = [ ("node_id", `String (id t)) ] in try let%bind logs = - match%map ci_http_request' ~access_token ~request_body with - | Ok - (Response.Platform_agnostic.Precomputed_block_dump (node_id, blocks)) - when String.equal node_id (id t) -> + match%map + run_command ~config:!config_path ~args "dump_precomputed_blocks" + with + | Ok (Response.Precomputed_block_dump (_, blocks)) -> blocks | _ -> failwith "invalid dump precomputed blocks response" @@ -1205,38 +1199,17 @@ module Workload_to_deploy = struct ?(has_archive_container = false) primary_container_id = "" - let get_nodes_from_workload t ~config = - let%bind cwd = Unix.getcwd () in - let open Malleable_error.Let_syntax in - let%bind app_id = - Deferred.bind ~f:Malleable_error.or_hard_error - (Integration_test_lib.Util.run_cmd_or_error cwd "kubectl" - ( base_kube_args config - @ [ "get" - ; "deployment" - ; t.workload_id - ; "-o" - ; "jsonpath={.spec.selector.matchLabels.app}" - ] ) ) - in - let%map node_id_str = - Integration_test_lib.Util.run_cmd_or_hard_error cwd "kubectl" - ( base_kube_args config - @ [ "get"; "pod"; "-l"; "app=" ^ app_id; "-o"; "name" ] ) - in - let node_id = - String.split node_id_str ~on:'\n' - |> List.filter ~f:(Fn.compose not String.is_empty) - |> List.map ~f:(String.substr_replace_first ~pattern:"pod/" ~with_:"") - |> String.concat ~sep:"TODO:" - in + let[@warning "-27"] get_nodes_from_workload t ~config = + let node_id = "node_id" in + (* TODO: get from ci runner *) let node_type = Node_type.Seed_node in (* TODO: get from ci runner *) let network_keypair = None in (* TODO: get from ci runner *) let password = "password" in (* TODO: get from ci runner *) - { Node.app_id; node_id; node_type; config; network_keypair; password } + Malleable_error.return + { Node.node_id; node_type; config; network_keypair; password } end type t = From 59cc03b6ee0a51ef07cd70c6191d800cc2ac9e3c Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 1 Aug 2023 04:03:05 +0000 Subject: [PATCH 18/90] (engine) Start writing config-based commands --- .../ci_interaction.ml | 407 ++++++++++-------- 1 file changed, 216 insertions(+), 191 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 7986a26cd9f..12c6a1ee832 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -103,8 +103,7 @@ module Network_config = struct let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) = - let { requires_graphql - ; genesis_ledger + let { genesis_ledger ; block_producers ; snark_coordinator ; snark_worker_fee @@ -344,7 +343,7 @@ module Network_config = struct ; cluster_region ; k8s_context = cluster_id ; testnet_name - ; deploy_graphql_ingress = requires_graphql + ; deploy_graphql_ingress = true ; mina_image = images.mina ; mina_agent_image = images.user_agent ; mina_bots_image = images.bots @@ -412,36 +411,6 @@ module Node_id = struct type t = string [@@deriving eq, yojson] end -module Request = struct - module Platform_specific = struct - type t = - | Create_network of Network_config.t' - | Deploy_network of Network_id.t - | Destroy_network of Network_id.t - [@@deriving to_yojson] - end - - type start_node = - { node_id : Node_id.t; fresh_state : bool; commit_sha : string } - [@@deriving eq, yojson] - - module Platform_agnostic = struct - type t = - | Access_token - | Start_node of start_node - | Stop_node of Node_id.t - | Dump_archive_data of Node_id.t - | Dump_mina_logs of Node_id.t - | Dump_precomputed_blocks of Node_id.t - | Run_replayer of Node_id.t - [@@deriving to_yojson] - end - - module type S = sig - type t [@@deriving to_yojson] - end -end - module Node_type = struct type t = | Archive_node @@ -483,14 +452,6 @@ module Access_token = struct end module Response = struct - module Platform_specific = struct - type t = - | Network_created of Network_id.t - | Network_deployed of Network_deploy_response.t - | Network_destroyed - [@@deriving eq, of_yojson] - end - (** Logproc takes care of these logs so we bypass them here *) module Node_logs = struct type t = string [@@deriving eq] @@ -504,95 +465,227 @@ module Response = struct Error (Yojson.Safe.to_string x) end - module Platform_agnostic = struct - type t = - | Access_token of Access_token.t - | Node_started of Node_id.t - | Node_stopped of Node_id.t - | Archive_data_dump of Node_id.t * string - | Mina_logs_dump of Node_id.t * Node_logs.t - | Precomputed_block_dump of Node_id.t * string - | Replayer_run of Node_id.t * string - [@@deriving eq, of_yojson] - end - - module type S = sig - type t [@@deriving of_yojson] - end + type t = + | Network_created of Network_id.t + | Network_deployed of Network_deploy_response.t + | Network_destroyed + | Node_started of Node_id.t + | Node_stopped of Node_id.t + | Archive_data_dump of Node_id.t * string + | Mina_logs_dump of Node_id.t * Node_logs.t + | Precomputed_block_dump of Node_id.t * string + | Replayer_run of Node_id.t * string + [@@deriving eq, of_yojson] end -(** - The general workflow could look something like this: - https://www.notion.so/minafoundation/Lucy-CI-Interactions-e36b48ac52994cafbe1367548e02241d?pvs=4 - *) +module Arg_type = struct + let bool = "bool" -module Ci = struct - type t = { host : string; port : int } [@@deriving of_yojson] + let int = "int" - let ci : t Async.Ivar.t = Async.Ivar.create () - - let uri () = - let ci = Async.Ivar.value_exn ci in - Uri.make ~scheme:"http" ~host:ci.host ~port:ci.port () + let string = "string" end -let request_ci_access_token () : - Response.Platform_agnostic.t Deferred.Or_error.t = - failwith "request_ci_access_token" - -let post ~(access_token : Access_token.t) ~request_body = - let headers = - let open Cohttp in - Header.add_list (Header.init ()) - [ ("Authorization", access_token) - ; ("Accept", "application/json") - ; ("Content-Type", "application/json") - ] +exception Invalid_config of string + +let version config = + match Yojson.Safe.Util.member "version" config with + | `Int version -> + version + | _ -> + raise @@ Invalid_config "config version should be an int" + +let action name config = + let open Yojson.Safe in + match Util.member "actions" config with + | `List actions -> + List.find_exn actions ~f:(fun action -> + Util.member "name" action |> equal @@ `String name ) + | _ -> + raise @@ Invalid_config "config actions should be a list" + +let raw_cmd t = + match Yojson.Safe.Util.member "command" t with + | `String raw_cmd -> + raw_cmd + | _ -> + raise @@ Invalid_config "config command should be a string" + +let validate_arg_type arg_type (arg_value : Yojson.Safe.t) = + let ( = ) = String.equal in + match arg_value with + | `Bool _ -> + arg_type = Arg_type.bool + | `Int _ -> + arg_type = Arg_type.int + | `String _ -> + arg_type = Arg_type.string + | _ -> + failwith "unimplemented" + +let validate_args ~args ~action = + let open Yojson.Safe in + let[@warning "-8"] (`Assoc a_list : t) = Util.member "args" action in + let () = assert (List.(length a_list = length args)) in + let rec aux = function + | [] -> + true + | (arg_name, (`String arg_type : t)) :: rest -> + let arg_value = + List.find_map_exn args ~f:(fun (name, value) -> + Option.some_if (String.equal name arg_name) value ) + in + validate_arg_type arg_type arg_value && aux rest + | _ -> + assert false + in + aux a_list + +let strip_quotes s = String.substr_replace_all s ~pattern:"\"" ~with_:"" + +let rec interpolate_args ~args raw_cmd = + let res = + match args with + | [] -> + raw_cmd + | (arg, value) :: args -> + let pattern = sprintf "{{%s}}" arg in + interpolate_args ~args + @@ String.substr_replace_all raw_cmd ~pattern + ~with_:(Yojson.Safe.to_string value) in - Deferred.Or_error.try_with ~here:[%here] ~extract_exn:true (fun () -> - Cohttp_async.Client.post ~headers - ~body:(Cohttp_async.Body.of_string request_body) - (Ci.uri ()) ) - -let to_json ~response ~body_str = - match Cohttp.Code.code_of_status (Cohttp_async.Response.status response) with - | 200 -> - Deferred.return (Ok (Yojson.Safe.from_string body_str)) - | code -> - Deferred.return - (Error (Error.createf "Status code %d -- %s" code body_str)) - -let http (type a b) (module Req : Request.S with type t = a) - (module Res : Response.S with type t = b) ~access_token ~(request_body : a) - = - let request_body = Req.to_yojson request_body |> Yojson.Safe.to_string in - let open Deferred.Or_error.Let_syntax in - let%bind response, body = post ~access_token ~request_body in - let%bind body_str = - Cohttp_async.Body.to_string body |> Deferred.map ~f:Result.return + strip_quotes res + +let prog_and_args ~args action = + let raw_cmd = raw_cmd action in + let () = assert (validate_args ~args ~action) in + let cmd_list = interpolate_args ~args raw_cmd |> String.split ~on:' ' in + (List.hd_exn cmd_list, List.tl_exn cmd_list) + +(* TODO: how to report errors? *) +let[@warning "-27"] run_command ?(timeout_seconds = 1) ?(dir = ".") ~config + ~args cmd_name = + let open Yojson.Safe in + let config = from_file config in + let version = version config in + let action = action cmd_name config in + let prog, args = prog_and_args ~args action in + let%map output = + Integration_test_lib.Util.run_cmd_or_error_timeout ~timeout_seconds dir prog + args in - let%bind body_json = to_json ~response ~body_str in - let open Yojson.Safe.Util in - ( match (member "errors" body_json, member "data" body_json) with - | `Null, `Null -> - Error (Error.of_string "Empty response from http query") - | error, `Null -> - Error (Error.of_string @@ to_string error) - | _, raw_json -> - Res.of_yojson raw_json |> Result.map_error ~f:Error.of_string ) - |> Deferred.return - -let ci_http_request ~access_token ~request_body = - http - (module Request.Platform_specific) - (module Response.Platform_specific) - ~access_token ~request_body - -let ci_http_request' ~access_token ~request_body = - http - (module Request.Platform_agnostic) - (module Response.Platform_agnostic) - ~access_token ~request_body + match output with + | Ok output -> + Yojson.Safe.from_string output |> Response.of_yojson + | _ -> + let cmd = String.concat ~sep:" " (prog :: args) in + Error + (sprintf "Failed to run command: %s\nConfig version: %d\n" cmd version) + +module Unit_tests = struct + module Test_values = struct + let deploy_network_raw_cmd = + "minimina network deploy --network-id {{network_id}}" + + let start_node_raw_cmd = + "minimina start-node --node-id {{node_id}} --fresh-state {{fresh_state}} \ + --commit-sha {{commit_sha}}" + + let deploy_network_action = + sprintf + {| + { + "name": "deploy_network", + "args": { + "network_id": "string" + }, + "command": "%s" + } + |} + deploy_network_raw_cmd + |> String.strip + + let start_node_action = + sprintf + {| + { + "name": "start_node", + "args": { + "node_id": "string", + "fresh_state": "bool", + "commit_sha": "string" + }, + "command": "%s" + } + |} + start_node_raw_cmd + |> String.strip + + let config = + sprintf + {| + { + "version": 1, + "actions": [%s,%s] + } + |} + deploy_network_action start_node_action + |> Yojson.Safe.from_string + end + + open Test_values + + let%test_unit "version" = + let res = version config in + let expect = 1 in + assert (res = expect) + + let%test_unit "raw command" = + let res = raw_cmd @@ Yojson.Safe.from_string deploy_network_action in + assert (String.equal res deploy_network_raw_cmd) + + let%test_unit "validate arg types" = + let bool_value = `Bool true in + let int_value = `Int 42 in + let string_value = `String "hello" in + assert (validate_arg_type Arg_type.bool bool_value) ; + assert (validate_arg_type Arg_type.int int_value) ; + assert (validate_arg_type Arg_type.string string_value) + + let%test_unit "validate args" = + let action = action "start_node" config in + let args = + [ ("node_id", `String "node0") + ; ("fresh_state", `Bool true) + ; ("commit_sha", `String "0123456abcdef") + ] + in + assert (validate_args ~args ~action) + + let%test_unit "interpolate string args" = + let res = + interpolate_args + ~args:[ ("network_id", `String "network0") ] + deploy_network_raw_cmd + in + let expect = "minimina network deploy --network-id network0" in + assert (String.equal res expect) + + let%test_unit "prog and args" = + let action = Yojson.Safe.from_string start_node_action in + let args = + [ ("node_id", `String "node0") + ; ("fresh_state", `Bool true) + ; ("commit_sha", `String "0123456abcdef") + ] + in + let prog, res_args = prog_and_args ~args action in + let expect = + String.split ~on:' ' @@ interpolate_args ~args start_node_raw_cmd + in + assert (String.equal prog @@ List.hd_exn expect) ; + assert (List.equal String.equal res_args @@ List.tl_exn expect) +end module Request_unit_tests = struct let ( = ) = String.equal @@ -616,69 +709,10 @@ module Request_unit_tests = struct assert ( result = {|{"mina_automation_location":"loc","debug_arg":true,"genesis_keypairs":{},"constants":{"constraints":{"sub_windows_per_window":11,"ledger_depth":35,"work_delay":2,"block_window_duration_ms":180000,"transaction_capacity_log_2":7,"pending_coinbase_depth":5,"coinbase_amount":"720000000000","supercharged_coinbase_factor":1,"account_creation_fee":"1","fork":null},"genesis":{"protocol":{"k":290,"slots_per_epoch":7140,"slots_per_sub_window":7,"delta":0,"genesis_state_timestamp":"2020-09-16 10:15:00.000000Z"},"txpool_max_size":3000,"num_accounts":null,"zkapp_proof_update_cost":10.26,"zkapp_signed_single_update_cost":9.140000000000001,"zkapp_signed_pair_update_cost":10.08,"zkapp_transaction_cost_limit":69.45,"max_event_elements":100,"max_action_elements":100}}}|} ) - - let%test_unit "Deploy network request" = - let open Request.Platform_specific in - let result = - Deploy_network "network0" |> to_yojson |> Yojson.Safe.to_string - in - assert (result = {|["Deploy_network","network0"]|}) - - let%test_unit "Destroy network request" = - let open Request.Platform_specific in - let result = - Destroy_network "network0" |> to_yojson |> Yojson.Safe.to_string - in - assert (result = {|["Destroy_network","network0"]|}) - - let%test_unit "Access token request" = - let open Request.Platform_agnostic in - let result = Access_token |> to_yojson |> Yojson.Safe.to_string in - assert (result = {|["Access_token"]|}) - - let%test_unit "Start node request" = - let open Request.Platform_agnostic in - let result = - Start_node - { node_id = "node0"; fresh_state = true; commit_sha = "0123456" } - |> to_yojson |> Yojson.Safe.to_string - in - assert ( - result - = {|["Start_node",{"node_id":"node0","fresh_state":true,"commit_sha":"0123456"}]|} ) - - let%test_unit "Stop node request" = - let open Request.Platform_agnostic in - let result = Stop_node "node0" |> to_yojson |> Yojson.Safe.to_string in - assert (result = {|["Stop_node","node0"]|}) - - let%test_unit "Dump archive data request" = - let open Request.Platform_agnostic in - let result = - Dump_archive_data "node0" |> to_yojson |> Yojson.Safe.to_string - in - assert (result = {|["Dump_archive_data","node0"]|}) - - let%test_unit "Dump mina logs request" = - let open Request.Platform_agnostic in - let result = Dump_mina_logs "node0" |> to_yojson |> Yojson.Safe.to_string in - assert (result = {|["Dump_mina_logs","node0"]|}) - - let%test_unit "Dump precomputed blocks request" = - let open Request.Platform_agnostic in - let result = - Dump_precomputed_blocks "node0" |> to_yojson |> Yojson.Safe.to_string - in - assert (result = {|["Dump_precomputed_blocks","node0"]|}) - - let%test_unit "Run replayer request" = - let open Request.Platform_agnostic in - let result = Run_replayer "node0" |> to_yojson |> Yojson.Safe.to_string in - assert (result = {|["Run_replayer","node0"]|}) end module Response_unit_tests = struct - open Response.Platform_specific + open Response let ( = ) = equal @@ -721,17 +755,8 @@ module Response_unit_tests = struct in assert (result = Network_destroyed) - open Response.Platform_agnostic - let ( = ) = equal - let%test_unit "Parse access token response" = - let result = - {|["Access_token","token0"]|} |> Yojson.Safe.from_string |> of_yojson - |> Result.ok_or_failwith - in - assert (result = Access_token "token0") - let%test_unit "Node started response" = let result = {|["Node_started","node0"]|} |> Yojson.Safe.from_string |> of_yojson From d5315d97b33c1a6113fde18e81158fc606adfeb3 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 2 Aug 2023 03:25:37 +0000 Subject: [PATCH 19/90] (test exec) Config validation for abstract engine --- src/app/test_executive/test_executive.ml | 57 ++++++++++----------- src/lib/integration_test_lib/test_config.ml | 55 +++++++++++++++++--- 2 files changed, 74 insertions(+), 38 deletions(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index 5b8892c0fb5..4fedf9b7a64 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -30,7 +30,7 @@ type inputs = ; mina_image : string ; archive_image : string option ; debug : bool - ; config_path : string + ; config_path : string option } let validate_inputs ~logger inputs (test_config : Test_config.t) : @@ -45,13 +45,22 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : "This test uses archive nodes. archive-image argument cannot be absent \ for this test" ; exit 1 ) - else Deferred.unit + else + let (Test_inputs_with_cli_inputs ((module Inputs), _)) = + inputs.test_inputs + in + if + String.equal Inputs.Engine.name "abstract" + && Option.is_none inputs.config_path + then ( + [%log fatal] "Must provide a config file when using the abstract engine" ; + exit 1 ) + else Deferred.unit let engines : engine list = - let abstract_engine = - (module Integration_test_abstract_engine : Intf.Engine.S) - in - [ ("cloud", abstract_engine); ("local", abstract_engine) ] + [ ("abstract", (module Integration_test_abstract_engine : Intf.Engine.S)) + ; ("cloud", (module Integration_test_cloud_engine : Intf.Engine.S)) + ] let tests : test list = [ ( "peers-reliability" @@ -116,9 +125,10 @@ let report_test_errors ~log_error_set ~internal_error_set = List.fold severities ~init:`None ~f:max_sev in let combine_errors error_set = - Error_accumulator.combine - [ Error_accumulator.map error_set.soft_errors ~f:(fun err -> (`Soft, err)) - ; Error_accumulator.map error_set.hard_errors ~f:(fun err -> (`Hard, err)) + let open Error_accumulator in + combine + [ map error_set.soft_errors ~f:(fun err -> (`Soft, err)) + ; map error_set.hard_errors ~f:(fun err -> (`Hard, err)) ] in let internal_errors = combine_errors internal_error_set in @@ -243,8 +253,7 @@ let dispatch_cleanup ~logger ~pause_cleanup_func ~network_cleanup_func deferred let test_name (test : test) - (inputs : (module Integration_test_lib.Intf.Test.Inputs_intf)) = - let (module Inputs) = inputs in + (module Inputs : Integration_test_lib.Intf.Test.Inputs_intf) = let module Test = (val test) (Inputs) in Test.test_name @@ -257,15 +266,8 @@ let main inputs = let test_name = test_name inputs.test (module Test_inputs) in let module T = Test (Test_inputs) in let logger = Logger.create () in - (* TODO: provide as json file *) let images = - { Test_config.Container_images.mina = inputs.mina_image - ; archive_node = - Option.value inputs.archive_image ~default:"archive_image_unused" - ; user_agent = "codaprotocol/coda-user-agent:0.1.5" - ; bots = "minaprotocol/mina-bots:latest" - ; points = "codaprotocol/coda-points-hack:32b.4" - } + Test_config.Container_images.mk inputs.mina_image inputs.archive_image in let%bind () = validate_inputs ~logger inputs T.config in [%log trace] "expanding network config" ; @@ -423,7 +425,7 @@ let config_path_arg = let doc = "Path to the CI config file." in let env = Arg.env_var "MINA_CI_CONFIG_PATH" ~doc in Arg.( - required + value & opt (some non_dir_file) None & info [ "config-path"; "config" ] ~env ~docv:"MINA_CI_CONFIG_PATH" ~doc) @@ -435,14 +437,6 @@ let mina_image_arg = & opt (some string) None & info [ "mina-image" ] ~env ~docv:"MINA_IMAGE" ~doc) -let images_arg = - let doc = "Identifier of the archive node docker image to test." in - let env = Arg.env_var "ARCHIVE_IMAGE" ~doc in - Arg.( - value - & opt (some string) None - & info [ "archive-image" ] ~env ~docv:"ARCHIVE_IMAGE" ~doc) - let archive_image_arg = let doc = "Identifier of the archive node docker image to test." in let env = Arg.env_var "ARCHIVE_IMAGE" ~doc in @@ -463,10 +457,10 @@ let help_term = Term.(ret @@ const (`Help (`Plain, None))) let info engine_name = let doc = match engine_name with + | "abstract" -> + "Run mina integration tests with the abstract engine." | "cloud" -> "Run mina integration tests on a remote cloud provider." - | "local" -> - "Run mina integration tests locally with Minimina." | _ -> assert false in @@ -476,7 +470,8 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = let info = info engine_name in let module Inputs = Make_test_inputs (Engine) () in let set_config path = - Engine.Network.config_path := path ; + (* TODO: deal with path relativity? *) + Option.iter path ~f:(fun p -> Engine.Network.config_path := p) ; path in let test_inputs_with_cli_inputs_arg = diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 104aca2dc7c..49e2281b86d 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -8,6 +8,49 @@ module Container_images = struct ; bots : string ; points : string } + + let required_value field json ~fail = + match Yojson.Safe.Util.member field json with + | `String value -> + value + | _ -> + failwith fail + + let optional_value field json ~default = + match Yojson.Safe.Util.member field json with + | `String value -> + value + | `Null -> + default + | _ -> + failwithf "%s image parse error\n" field () + + let mina_images path = + let json = Yojson.Safe.from_file path in + let mina = required_value "mina" json ~fail:"Must provide mina image" in + let archive_node = + optional_value "archive_node" json ~default:"archive_image_unused" + in + let user_agent = + optional_value "user_agent" json + ~default:"codaprotocol/coda-user-agent:0.1.5" + in + let bots = + optional_value "bots" json ~default:"minaprotocol/mina-bots:latest" + in + let points = + optional_value "points" json + ~default:"codaprotocol/coda-points-hack:32b.4" + in + { mina; archive_node; user_agent; bots; points } + + let mk mina archive_node = + { mina + ; archive_node = Option.value archive_node ~default:"archive_image_unused" + ; user_agent = "codaprotocol/coda-user-agent:0.1.5" + ; bots = "minaprotocol/mina-bots:latest" + ; points = "codaprotocol/coda-points-hack:32b.4" + } end module Test_Account = struct @@ -16,6 +59,7 @@ module Test_Account = struct ; balance : string ; timing : Mina_base.Account_timing.t } + [@@deriving to_yojson] end module Epoch_data = struct @@ -28,7 +72,7 @@ module Epoch_data = struct end module Block_producer_node = struct - type t = { node_name : string; account_name : string } + type t = { node_name : string; account_name : string } [@@deriving to_yojson] end module Snark_coordinator_node = struct @@ -44,17 +88,13 @@ type constants = type t = { requires_graphql : bool - (* temporary flag to enable/disable graphql ingress deployments *) - (* testnet topography *) ; genesis_ledger : Test_Account.t list ; epoch_data : Epoch_data.t option ; block_producers : Block_producer_node.t list ; snark_coordinator : Snark_coordinator_node.t option ; snark_worker_fee : string ; num_archive_nodes : int - ; log_precomputed_blocks : bool - (* ; num_plain_nodes : int *) - (* blockchain constants *) + ; log_precomputed_blocks : bool (* blockchain constants *) ; proof_config : Runtime_config.Proof_keys.t ; k : int ; delta : int @@ -62,6 +102,7 @@ type t = ; slots_per_sub_window : int ; txpool_max_size : int } +[@@deriving to_yojson] let proof_config_default : Runtime_config.Proof_keys.t = { level = Some Full @@ -86,7 +127,7 @@ let default = ; snark_coordinator = None ; snark_worker_fee = "0.025" ; num_archive_nodes = 0 - ; log_precomputed_blocks = false (* ; num_plain_nodes = 0 *) + ; log_precomputed_blocks = false ; proof_config = proof_config_default ; k = 20 ; slots_per_epoch = 3 * 8 * 20 From 8c4ffd1f934f6e21f59f78df0ebbc0f917753260 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 2 Aug 2023 03:28:33 +0000 Subject: [PATCH 20/90] (config) Update parsing and unit tests --- .../ci_interaction.ml | 647 +++++++----------- .../config.json | 39 ++ .../unit_tests.ml | 290 ++++++++ .../integration_test_lib/network_keypair.ml | 2 +- src/lib/integration_test_lib/util.ml | 26 +- 5 files changed, 606 insertions(+), 398 deletions(-) create mode 100644 src/lib/integration_test_abstract_engine/config.json create mode 100644 src/lib/integration_test_abstract_engine/unit_tests.ml diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 12c6a1ee832..52d27282950 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -5,20 +5,6 @@ open Signature_lib open Mina_base open Integration_test_lib -let aws_region = "us-west-2" - -let aws_route53_zone_id = "ZJPR9NA6W9M7F" - -let project_id = "o1labs-192920" - -let cluster_id = "gke_o1labs-192920_us-west1_mina-integration-west1" - -let cluster_name = "mina-integration-west1" - -let cluster_region = "us-west1" - -let cluster_zone = "us-west1a" - module Network_config = struct module Cli_inputs = Cli_inputs @@ -30,19 +16,15 @@ module Network_config = struct { name : string; public_key : string; worker_nodes : int } [@@deriving to_yojson] - type terraform_config = - { k8s_context : string - ; cluster_name : string - ; cluster_region : string - ; aws_route53_zone_id : string - ; testnet_name : string + type config = + { network_id : string + ; config_dir : string ; deploy_graphql_ingress : bool ; mina_image : string ; mina_agent_image : string ; mina_bots_image : string ; mina_points_image : string ; mina_archive_image : string - (* this field needs to be sent as a string to terraform, even though it's a json encoded value *) ; runtime_config : Yojson.Safe.t [@to_yojson fun j -> `String (Yojson.Safe.to_string j)] ; block_producer_configs : block_producer_config list @@ -52,17 +34,12 @@ module Network_config = struct ; mina_archive_schema_aux_files : string list ; snark_coordinator_config : snark_coordinator_config option ; snark_worker_fee : string - ; cpu_request : int - ; mem_request : string - ; worker_cpu_request : int - ; worker_mem_request : string } [@@deriving to_yojson] (* TODO: replace with t' *) type t = - { mina_automation_location : string - ; debug_arg : bool + { debug_arg : bool ; genesis_keypairs : (Network_keypair.t Core.String.Map.t [@to_yojson @@ -73,14 +50,13 @@ module Network_config = struct (k, Network_keypair.to_yojson v) :: accum ) map )] ) ; constants : Test_config.constants - ; terraform : terraform_config + ; config : config } [@@deriving to_yojson] (* TODO: remove *) type t' = - { mina_automation_location : string - ; debug_arg : bool + { debug_arg : bool ; genesis_keypairs : (Network_keypair.t Core.String.Map.t [@to_yojson @@ -94,16 +70,13 @@ module Network_config = struct } [@@deriving to_yojson] - let terraform_config_to_assoc t = - let[@warning "-8"] (`Assoc assoc : Yojson.Safe.t) = - terraform_config_to_yojson t - in - assoc + let testnet_log_filter t = t.config.network_id let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) = - let { genesis_ledger + let { requires_graphql + ; genesis_ledger ; block_producers ; snark_coordinator ; snark_worker_fee @@ -125,8 +98,7 @@ module Network_config = struct let user_len = Int.min 5 (String.length user_sanitized) in let user = String.sub user_sanitized ~pos:0 ~len:user_len in let git_commit = Mina_version.commit_id_short in - (* see ./src/app/test_executive/README.md for information regarding the namespace name format and length restrictions *) - let testnet_name = "it-" ^ user ^ "-" ^ git_commit ^ "-" ^ test_name in + let network_id = "it-" ^ user ^ "-" ^ git_commit ^ "-" ^ test_name in (* check to make sure the test writer hasn't accidentally created duplicate names of accounts and keys *) let key_names_list = List.map genesis_ledger ~f:(fun acct -> acct.account_name) @@ -334,16 +306,13 @@ module Network_config = struct in (* NETWORK CONFIG *) - { mina_automation_location = cli_inputs.mina_automation_location - ; debug_arg = debug + { debug_arg = debug ; genesis_keypairs ; constants - ; terraform = - { cluster_name - ; cluster_region - ; k8s_context = cluster_id - ; testnet_name - ; deploy_graphql_ingress = true + ; config = + { network_id + ; config_dir = cli_inputs.mina_automation_location + ; deploy_graphql_ingress = requires_graphql ; mina_image = images.mina ; mina_agent_image = images.user_agent ; mina_bots_image = images.bots @@ -357,50 +326,25 @@ module Network_config = struct ; mina_archive_schema_aux_files ; snark_coordinator_config ; snark_worker_fee - ; aws_route53_zone_id - ; cpu_request = 6 - ; mem_request = "12Gi" - ; worker_cpu_request = 6 - ; worker_mem_request = "8Gi" } } +end - let to_terraform network_config = - let open Terraform in - [ Block.Terraform - { Block.Terraform.required_version = ">= 0.12.0" - ; backend = - Backend.Local - { path = - "terraform-" ^ network_config.terraform.testnet_name - ^ ".tfstate" - } - } - ; Block.Provider - { Block.Provider.provider = "google" - ; region = cluster_region - ; zone = Some cluster_zone - ; project = Some project_id - ; alias = None - } - ; Block.Module - { Block.Module.local_name = "integration_testnet" - ; providers = [ ("google.gke", "google") ] - ; source = "../../modules/o1-integration" - ; args = terraform_config_to_assoc network_config.terraform - } - ] - - let testnet_log_filter network_config = - Printf.sprintf - {| - resource.labels.project_id="%s" - resource.labels.location="%s" - resource.labels.cluster_name="%s" - resource.labels.namespace_name="%s" - |} - project_id cluster_region cluster_name - network_config.terraform.testnet_name +module Config_file = struct + type t = { version : int; actions : action list } [@@deriving eq, yojson] + + and action = { name : string; args : Yojson.Safe.t; command : string } + [@@deriving eq, yojson] + + exception Invalid_version of Yojson.Safe.t + + exception Invalid_actions of Yojson.Safe.t + + exception Invalid_args of Yojson.Safe.t + + exception Invalid_arg_type_should_be_string of string * Yojson.Safe.t + + exception Invalid_command of Yojson.Safe.t end module Network_id = struct @@ -423,77 +367,187 @@ module Node_type = struct let to_string nt = to_yojson nt |> Yojson.Safe.to_string let of_string s = of_yojson @@ `List [ `String s ] |> Result.ok_or_failwith + + let is_archive_node = function Archive_node -> true | _ -> false + + let is_block_producer = function Block_producer_node -> true | _ -> false + + let is_seed_node = function Seed_node -> true | _ -> false + + let is_snark_worker = function Snark_worker -> true | _ -> false + + let is_snark_coordinator = function Snark_coordinator -> true | _ -> false +end + +(** Logproc takes care of these logs so we bypass them here *) +module Node_logs = struct + type t = string [@@deriving eq] + + let to_yojson s = `String s + + let of_yojson = function + | `String s -> + Ok s + | x -> + Error (Yojson.Safe.to_string x) +end + +module Network_created = struct + type t = { network_id : Network_id.t } [@@deriving eq, of_yojson] end -module Network_deploy_response = struct +module Network_deployed = struct module Map = Map.Make (String) include Map - type t = (Node_type.t * string option) Map.t [@@deriving eq] + exception Invalid_entry of string * Yojson.Safe.t + + exception Invalid_output of string + + exception Invalid_keypair of Yojson.Safe.t + + type t = node_info Map.t [@@deriving eq] + + and node_info = + { node_type : Node_type.t + ; network_keypair : Network_keypair.t option + ; password : string + ; graphql_uri : string + } + [@@deriving yojson] + + let network_keypair_of_yojson (sk : Yojson.Safe.t) = + match sk with + | `Null -> + None + | `Assoc + [ ("private_key", `String private_key) + ; ("privkey_password", `String privkey_password) + ; ("keypair_name", `String keypair_name) + ] -> + let keypair = + Keypair.of_private_key_exn + @@ Private_key.of_base58_check_exn private_key + in + Some + Network_keypair. + { keypair + ; keypair_name + ; privkey_password + ; private_key + ; public_key = + Public_key.to_bigstring keypair.public_key + |> Bigstring.to_string + } + | _ -> + raise @@ Invalid_keypair sk let of_yojson = let f accum = function - | node_id, `Tuple [ `String nt; `String gql_uri ] -> - Map.set accum ~key:node_id ~data:(Node_type.of_string nt, Some gql_uri) - | node_id, `Tuple [ `String nt; `Null ] -> - Map.set accum ~key:node_id ~data:(Node_type.of_string nt, None) - | _ -> - failwith "invalid network_deploy_response yojson entry" + | ( node_id + , `Assoc + [ ("node_type", `String nt) + ; ("network_keypair", keypair) + ; ("password", `String password) + ; ("graphql_uri", `String graphql_uri) + ] ) -> + Map.set accum ~key:node_id + ~data: + { node_type = Node_type.of_string nt + ; network_keypair = network_keypair_of_yojson keypair + ; password + ; graphql_uri + } + | node_id, t -> + raise @@ Invalid_entry (node_id, t) in function | `Assoc a_list -> Ok (List.fold a_list ~init:(Map.empty : t) ~f) - | _ -> - Error "invalid network_deploy_response yojson" + | t -> + raise @@ Invalid_output (Yojson.Safe.to_string t) end -module Access_token = struct - type t = string [@@deriving eq, yojson] +module Node_started = struct + type t = { node_id : Node_id.t } [@@deriving eq, of_yojson] + + exception Invalid_response of string end -module Response = struct - (** Logproc takes care of these logs so we bypass them here *) - module Node_logs = struct - type t = string [@@deriving eq] +module Node_stopped = struct + type t = { node_id : Node_id.t } [@@deriving eq, of_yojson] + + exception Invalid_response of string +end - let to_yojson s = `String s +module Archive_data_dump = struct + type t = { node_id : Node_id.t; data : string } [@@deriving eq, of_yojson] - let of_yojson = function - | `String s -> - Ok s - | x -> - Error (Yojson.Safe.to_string x) - end + exception Invalid_response of string +end +module Mina_logs_dump = struct + type t = { node_id : Node_id.t; logs : Node_logs.t } + [@@deriving eq, of_yojson] + + exception Invalid_response of string +end + +module Precomputed_block_dump = struct + type t = { node_id : Node_id.t; blocks : string } [@@deriving eq, of_yojson] + + exception Invalid_response of string +end + +module Replayer_run = struct + type t = { node_id : Node_id.t; logs : Node_logs.t } + [@@deriving eq, of_yojson] + + exception Invalid_response of string +end + +module Command_output = struct type t = - | Network_created of Network_id.t - | Network_deployed of Network_deploy_response.t + | Network_created of Network_created.t + | Network_deployed of Network_deployed.t | Network_destroyed - | Node_started of Node_id.t - | Node_stopped of Node_id.t - | Archive_data_dump of Node_id.t * string - | Mina_logs_dump of Node_id.t * Node_logs.t - | Precomputed_block_dump of Node_id.t * string - | Replayer_run of Node_id.t * string + | Node_started of Node_started.t + | Node_stopped of Node_stopped.t + | Archive_data_dump of Archive_data_dump.t + | Mina_logs_dump of Mina_logs_dump.t + | Precomputed_block_dump of Precomputed_block_dump.t + | Replayer_run of Replayer_run.t [@@deriving eq, of_yojson] end module Arg_type = struct + (* TODO: what other arg types? *) let bool = "bool" let int = "int" let string = "string" -end -exception Invalid_config of string + exception Invalid_arg_type of Yojson.Safe.t + + let arg_type (t : Yojson.Safe.t) = + match t with + | `Bool _ -> + bool + | `Int _ -> + int + | `String _ -> + string + | _ -> + raise @@ Invalid_arg_type t +end let version config = match Yojson.Safe.Util.member "version" config with | `Int version -> version - | _ -> - raise @@ Invalid_config "config version should be an int" + | t -> + raise @@ Config_file.Invalid_version t let action name config = let open Yojson.Safe in @@ -501,302 +555,111 @@ let action name config = | `List actions -> List.find_exn actions ~f:(fun action -> Util.member "name" action |> equal @@ `String name ) - | _ -> - raise @@ Invalid_config "config actions should be a list" + | t -> + raise @@ Config_file.Invalid_actions t + +let args_of_action action = + let open Yojson.Safe in + let a_list = + match Util.member "args" action with + | `Assoc a_list -> + a_list + | t -> + raise @@ Config_file.Invalid_args t + in + List.map a_list ~f:(function + | name, `String type_name -> + (name, type_name) + | name, t -> + raise @@ Config_file.Invalid_arg_type_should_be_string (name, t) ) let raw_cmd t = match Yojson.Safe.Util.member "command" t with | `String raw_cmd -> raw_cmd - | _ -> - raise @@ Invalid_config "config command should be a string" - -let validate_arg_type arg_type (arg_value : Yojson.Safe.t) = - let ( = ) = String.equal in - match arg_value with - | `Bool _ -> - arg_type = Arg_type.bool - | `Int _ -> - arg_type = Arg_type.int - | `String _ -> - arg_type = Arg_type.string - | _ -> - failwith "unimplemented" + | cmd -> + raise @@ Config_file.Invalid_command cmd + +let validate_arg_type arg_typ arg_value = + String.equal arg_typ @@ Arg_type.arg_type arg_value + +exception Invalid_config_args_num + +exception Missing_config_arg of string * string + +exception Invalid_config_arg_type of string * Yojson.Safe.t * string let validate_args ~args ~action = - let open Yojson.Safe in - let[@warning "-8"] (`Assoc a_list : t) = Util.member "args" action in - let () = assert (List.(length a_list = length args)) in + let arg_list = args_of_action action in + let () = + if not List.(length arg_list = length args) then + raise @@ Invalid_config_args_num + in let rec aux = function | [] -> true - | (arg_name, (`String arg_type : t)) :: rest -> + | (arg_name, arg_type) :: rest -> let arg_value = - List.find_map_exn args ~f:(fun (name, value) -> - Option.some_if (String.equal name arg_name) value ) + try + List.find_map_exn args ~f:(fun (name, value) -> + Option.some_if (String.equal name arg_name) value ) + with Not_found_s _ -> + raise @@ Missing_config_arg (arg_name, arg_type) in - validate_arg_type arg_type arg_value && aux rest - | _ -> - assert false + if validate_arg_type arg_type arg_value then aux rest + else raise @@ Invalid_config_arg_type (arg_name, arg_value, arg_type) in - aux a_list - -let strip_quotes s = String.substr_replace_all s ~pattern:"\"" ~with_:"" + aux arg_list let rec interpolate_args ~args raw_cmd = - let res = - match args with - | [] -> - raw_cmd - | (arg, value) :: args -> - let pattern = sprintf "{{%s}}" arg in - interpolate_args ~args - @@ String.substr_replace_all raw_cmd ~pattern - ~with_:(Yojson.Safe.to_string value) - in - strip_quotes res + match args with + | [] -> + raw_cmd + | (arg, value) :: args -> + let pattern = sprintf "{{%s}}" arg in + interpolate_args ~args + @@ String.substr_replace_all raw_cmd ~pattern + ~with_:(Yojson.Safe.to_string value |> Util.drop_outer_quotes) let prog_and_args ~args action = let raw_cmd = raw_cmd action in - let () = assert (validate_args ~args ~action) in let cmd_list = interpolate_args ~args raw_cmd |> String.split ~on:' ' in (List.hd_exn cmd_list, List.tl_exn cmd_list) -(* TODO: how to report errors? *) -let[@warning "-27"] run_command ?(timeout_seconds = 1) ?(dir = ".") ~config - ~args cmd_name = - let open Yojson.Safe in - let config = from_file config in +let run_command ?(suppress_logs = false) ?(timeout_seconds = 1) ?(dir = ".") + ~config ~args cmd_name = + let config = Yojson.Safe.from_file config in let version = version config in let action = action cmd_name config in - let prog, args = prog_and_args ~args action in + let action_args = args_of_action action in + let () = assert (validate_args ~args ~action) in + let prog, arg_values = prog_and_args ~args action in + let cmd = String.concat ~sep:" " (prog :: arg_values) in let%map output = - Integration_test_lib.Util.run_cmd_or_error_timeout ~timeout_seconds dir prog - args + Util.run_cmd_or_error_timeout ~suppress_logs ~timeout_seconds dir prog + arg_values in match output with | Ok output -> - Yojson.Safe.from_string output |> Response.of_yojson + if not suppress_logs then + [%log' spam (Logger.create ())] + "Successful command execution\nCommand: %s\nOutput: %s" cmd output ; + Ok output | _ -> - let cmd = String.concat ~sep:" " (prog :: args) in + if not suppress_logs then + [%log' error (Logger.create ())] "Failed to run command: %s" cmd ; Error - (sprintf "Failed to run command: %s\nConfig version: %d\n" cmd version) - -module Unit_tests = struct - module Test_values = struct - let deploy_network_raw_cmd = - "minimina network deploy --network-id {{network_id}}" - - let start_node_raw_cmd = - "minimina start-node --node-id {{node_id}} --fresh-state {{fresh_state}} \ - --commit-sha {{commit_sha}}" - - let deploy_network_action = - sprintf - {| - { - "name": "deploy_network", - "args": { - "network_id": "string" - }, - "command": "%s" - } - |} - deploy_network_raw_cmd - |> String.strip - - let start_node_action = - sprintf - {| - { - "name": "start_node", - "args": { - "node_id": "string", - "fresh_state": "bool", - "commit_sha": "string" - }, - "command": "%s" - } - |} - start_node_raw_cmd - |> String.strip - - let config = - sprintf - {| - { - "version": 1, - "actions": [%s,%s] - } - |} - deploy_network_action start_node_action - |> Yojson.Safe.from_string - end - - open Test_values - - let%test_unit "version" = - let res = version config in - let expect = 1 in - assert (res = expect) - - let%test_unit "raw command" = - let res = raw_cmd @@ Yojson.Safe.from_string deploy_network_action in - assert (String.equal res deploy_network_raw_cmd) - - let%test_unit "validate arg types" = - let bool_value = `Bool true in - let int_value = `Int 42 in - let string_value = `String "hello" in - assert (validate_arg_type Arg_type.bool bool_value) ; - assert (validate_arg_type Arg_type.int int_value) ; - assert (validate_arg_type Arg_type.string string_value) - - let%test_unit "validate args" = - let action = action "start_node" config in - let args = - [ ("node_id", `String "node0") - ; ("fresh_state", `Bool true) - ; ("commit_sha", `String "0123456abcdef") - ] - in - assert (validate_args ~args ~action) - - let%test_unit "interpolate string args" = - let res = - interpolate_args - ~args:[ ("network_id", `String "network0") ] - deploy_network_raw_cmd - in - let expect = "minimina network deploy --network-id network0" in - assert (String.equal res expect) - - let%test_unit "prog and args" = - let action = Yojson.Safe.from_string start_node_action in - let args = - [ ("node_id", `String "node0") - ; ("fresh_state", `Bool true) - ; ("commit_sha", `String "0123456abcdef") - ] - in - let prog, res_args = prog_and_args ~args action in - let expect = - String.split ~on:' ' @@ interpolate_args ~args start_node_raw_cmd - in - assert (String.equal prog @@ List.hd_exn expect) ; - assert (List.equal String.equal res_args @@ List.tl_exn expect) -end - -module Request_unit_tests = struct - let ( = ) = String.equal - - let%test_unit "Create network request" = - let constants : Test_config.constants = - { constraints = Genesis_constants.Constraint_constants.compiled - ; genesis = Genesis_constants.compiled - } - in - let network_config : Network_config.t' = - { mina_automation_location = "loc" - ; debug_arg = true - ; genesis_keypairs = Core.String.Map.empty - ; constants - } - in - let result = - Network_config.t'_to_yojson network_config |> Yojson.Safe.to_string - in - assert ( - result - = {|{"mina_automation_location":"loc","debug_arg":true,"genesis_keypairs":{},"constants":{"constraints":{"sub_windows_per_window":11,"ledger_depth":35,"work_delay":2,"block_window_duration_ms":180000,"transaction_capacity_log_2":7,"pending_coinbase_depth":5,"coinbase_amount":"720000000000","supercharged_coinbase_factor":1,"account_creation_fee":"1","fork":null},"genesis":{"protocol":{"k":290,"slots_per_epoch":7140,"slots_per_sub_window":7,"delta":0,"genesis_state_timestamp":"2020-09-16 10:15:00.000000Z"},"txpool_max_size":3000,"num_accounts":null,"zkapp_proof_update_cost":10.26,"zkapp_signed_single_update_cost":9.140000000000001,"zkapp_signed_pair_update_cost":10.08,"zkapp_transaction_cost_limit":69.45,"max_event_elements":100,"max_action_elements":100}}}|} ) -end - -module Response_unit_tests = struct - open Response - - let ( = ) = equal - - let%test_unit "Parse network created response" = - let result = - `List [ `String "Network_created"; `String "node0" ] - |> of_yojson |> Result.ok_or_failwith - in - assert (result = Network_created "node0") - - let%test_unit "Parse network deployed response" = - let open Node_type in - let open Network_deploy_response in - let result = - {| - { "node0": ("Archive_node", "gql0") - , "node1": ("Block_producer_node", "gql1") - , "node2": ("Seed_node", "gql2") - , "node3": ("Snark_worker", null) - , "node4": ("Snark_coordinator", "gql4") - } - |} - |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith - in - let ( = ) = equal in - assert ( - result - = of_alist_exn - [ ("node0", (Archive_node, Some "gql0")) - ; ("node1", (Block_producer_node, Some "gql1")) - ; ("node2", (Seed_node, Some "gql2")) - ; ("node3", (Snark_worker, None)) - ; ("node4", (Snark_coordinator, Some "gql4")) - ] ) - - let%test_unit "Parse network destroyed response" = - let result = - {|["Network_destroyed"]|} |> Yojson.Safe.from_string |> of_yojson - |> Result.ok_or_failwith - in - assert (result = Network_destroyed) - - let ( = ) = equal - - let%test_unit "Node started response" = - let result = - {|["Node_started","node0"]|} |> Yojson.Safe.from_string |> of_yojson - |> Result.ok_or_failwith - in - assert (result = Node_started "node0") - - let%test_unit "Node stopped response" = - let result = - {|["Node_stopped","node0"]|} |> Yojson.Safe.from_string |> of_yojson - |> Result.ok_or_failwith - in - assert (result = Node_stopped "node0") - - let%test_unit "Archive data dump response" = - let result = - {|["Archive_data_dump","node0","data0"]|} |> Yojson.Safe.from_string - |> of_yojson |> Result.ok_or_failwith - in - assert (result = Archive_data_dump ("node0", "data0")) - - let%test_unit "Mina logs dump response" = - let raw_logs = "{\"log0\":\"msg0\"}\n{\"log1\":\"msg1\"}" in - let result = - `List [ `String "Mina_logs_dump"; `String "node0"; `String raw_logs ] - |> of_yojson |> Result.ok_or_failwith - in - assert (result = Mina_logs_dump ("node0", raw_logs)) - - let%test_unit "Precomputed block dump response" = - let result = - {|["Precomputed_block_dump","node0","blocks"]|} |> Yojson.Safe.from_string - |> of_yojson |> Result.ok_or_failwith - in - assert (result = Precomputed_block_dump ("node0", "blocks")) - - let%test_unit "Replayer run response" = - let result = - {|["Replayer_run","node0","logs0"]|} |> Yojson.Safe.from_string - |> of_yojson |> Result.ok_or_failwith - in - assert (result = Replayer_run ("node0", "logs0")) -end + (sprintf + "Failed to run command: %s\n\ + \ Config version: %d\n\ + \ Expected args types:\n\ + \ %s\n\ + \ Attempted args:\n\ + \ %s\n" + cmd version + ( String.concat ~sep:", " + @@ List.map action_args ~f:(fun (name, type_name) -> + name ^ ": " ^ type_name ) ) + ( String.concat ~sep:", " + @@ List.map args ~f:(fun (name, value) -> + name ^ ": " ^ Yojson.Safe.to_string value ) ) ) diff --git a/src/lib/integration_test_abstract_engine/config.json b/src/lib/integration_test_abstract_engine/config.json new file mode 100644 index 00000000000..afe6a914eb1 --- /dev/null +++ b/src/lib/integration_test_abstract_engine/config.json @@ -0,0 +1,39 @@ +{ + "version": 1, + "actions": [ + { + "name": "start_network", + "args": {}, + "command": "minimina network start" + }, + { + "name": "get_network_status", + "args": { + "network_id": "string" + }, + "command": "minimina network status --network-id {{network_id}}" + }, + { + "name": "get_logs", + "args": { + "network_id": "string", + "node_id": "string" + }, + "command": "minimina node logs --node-id {{node_id}} --network-id {{network_id}}" + }, + { + "name": "echo", + "args": { + "msg": "string" + }, + "command": "echo \"{\"msg\":\"{{msg}}\"}\"" + }, + { + "name": "cat", + "args": { + "path": "string" + }, + "command": "cat {{path}}" + } + ] +} diff --git a/src/lib/integration_test_abstract_engine/unit_tests.ml b/src/lib/integration_test_abstract_engine/unit_tests.ml new file mode 100644 index 00000000000..b5651e95103 --- /dev/null +++ b/src/lib/integration_test_abstract_engine/unit_tests.ml @@ -0,0 +1,290 @@ +open Core_kernel +open Integration_test_lib +open Ci_interaction + +module Test_values = struct + let config_file = + (* cwd = ./_build/default/src/lib/integration_test_abstract_engine/ *) + let cwd_list = String.split ~on:'/' @@ Sys.getcwd () in + let dir_path = + List.take cwd_list (List.length cwd_list - 5) + @ [ "src/lib/integration_test_abstract_engine/config.json" ] + in + String.concat ~sep:"/" dir_path + + let deploy_network_raw_cmd = + "minimina network deploy --network-id {{network_id}}" + + let start_node_raw_cmd = + "minimina start-node --node-id {{node_id}} --fresh-state {{fresh_state}} \ + --commit-sha {{commit_sha}}" + + let deploy_network_action = + sprintf + {| + { + "name": "deploy_network", + "args": { + "network_id": "string" + }, + "command": "%s" + } + |} + deploy_network_raw_cmd + |> String.strip + + let start_node_action = + sprintf + {| + { + "name": "start_node", + "args": { + "node_id": "string", + "fresh_state": "bool", + "commit_sha": "string" + }, + "command": "%s" + } + |} + start_node_raw_cmd + |> String.strip + + let config = + sprintf + {| + { + "version": 1, + "actions": [%s,%s] + } + |} + deploy_network_action start_node_action + |> Yojson.Safe.from_string +end + +open Test_values + +module Helper_tests = struct + let%test_unit "version" = + let res = version config in + let expect = 1 in + assert (res = expect) + + let ( = ) = String.equal + + let%test_unit "raw command" = + let res = raw_cmd @@ Yojson.Safe.from_string deploy_network_action in + assert (res = deploy_network_raw_cmd) + + let%test_unit "validate arg types" = + let bool_value = `Bool true in + let int_value = `Int 42 in + let string_value = `String "hello" in + assert (validate_arg_type Arg_type.bool bool_value) ; + assert (validate_arg_type Arg_type.int int_value) ; + assert (validate_arg_type Arg_type.string string_value) + + let%test_unit "validate args" = + let action = action "start_node" config in + let args = + [ ("node_id", `String "node0") + ; ("fresh_state", `Bool true) + ; ("commit_sha", `String "0123456abcdef") + ] + in + assert (validate_args ~args ~action) + + let%test_unit "interpolate string args" = + let res = + interpolate_args + ~args:[ ("network_id", `String "network0") ] + deploy_network_raw_cmd + in + let expect = {|minimina network deploy --network-id network0|} in + assert (res = expect) + + let%test_unit "prog and args" = + let action = Yojson.Safe.from_string start_node_action in + let args = + [ ("node_id", `String "node0") + ; ("fresh_state", `Bool true) + ; ("commit_sha", `String "0123456abcdef") + ] + in + let prog, res_args = prog_and_args ~args action in + let expect = + String.split ~on:' ' @@ interpolate_args ~args start_node_raw_cmd + in + assert (prog = List.hd_exn expect) ; + assert (List.equal ( = ) res_args @@ List.tl_exn expect) +end + +module Run_command_tests = struct + module Arg_failures = struct + let ( = ) = String.equal + + let%test_unit "run command arg type validation failure" = + let arg, value = ("msg", `Int 42) in + try + ignore @@ run_command ~config:config_file ~args:[ (arg, value) ] "echo" + with Invalid_config_arg_type (arg_name, arg_value, arg_type) -> + assert ( + arg_name = "msg" && arg_type = Arg_type.string + && Yojson.Safe.equal arg_value value ) + + let%test_unit "run command arg number failure" = + try + ignore + @@ run_command ~config:config_file + ~args:[ ("msg", `String "hello"); ("num", `Int 42) ] + "echo" + with Invalid_config_args_num -> () + + let%test_unit "run command missing arg failure" = + try + ignore + @@ run_command ~config:config_file + ~args:[ ("msg0", `String "hello") ] + "echo" + with Missing_config_arg (arg_name, arg_type) -> + assert (arg_name = "msg" && arg_type = Arg_type.string) + end + + module Successful = struct + open Async.Deferred.Let_syntax + + let echo_command = + let arg, value = ("msg", `String "hello") in + let%map output = + run_command ~suppress_logs:true ~config:config_file + ~args:[ (arg, value) ] + "echo" + in + if Result.is_error output then assert false + + let cat_command = + let arg, value = ("path", `String config_file) in + match%map + run_command ~suppress_logs:true ~config:config_file + ~args:[ (arg, value) ] + "cat" + with + | Ok output -> + ignore + ( output |> Yojson.Safe.from_string |> Config_file.of_yojson + |> Result.ok_or_failwith ) + | Error _ -> + assert false + + let%test_unit "run command tests" = + ignore + (let%bind _ = Async.Deferred.all [ cat_command; echo_command ] in + exit 0 ) + end +end + +module Request_tests = struct + let ( = ) = String.equal + + let%test_unit "Create network request" = + let constants : Test_config.constants = + { constraints = Genesis_constants.Constraint_constants.compiled + ; genesis = Genesis_constants.compiled + } + in + let network_config : Network_config.t' = + { debug_arg = true; genesis_keypairs = Core.String.Map.empty; constants } + in + let result = + Network_config.t'_to_yojson network_config |> Yojson.Safe.to_string + in + assert ( + result + = {|{"debug_arg":true,"genesis_keypairs":{},"constants":{"constraints":{"sub_windows_per_window":11,"ledger_depth":35,"work_delay":2,"block_window_duration_ms":180000,"transaction_capacity_log_2":7,"pending_coinbase_depth":5,"coinbase_amount":"720000000000","supercharged_coinbase_factor":1,"account_creation_fee":"1","fork":null},"genesis":{"protocol":{"k":290,"slots_per_epoch":7140,"slots_per_sub_window":7,"delta":0,"genesis_state_timestamp":"2020-09-16 10:15:00.000000Z"},"txpool_max_size":3000,"num_accounts":null,"zkapp_proof_update_cost":10.26,"zkapp_signed_single_update_cost":9.140000000000001,"zkapp_signed_pair_update_cost":10.08,"zkapp_transaction_cost_limit":69.45,"max_event_elements":100,"max_action_elements":100}}}|} ) +end + +module Parse_output_tests = struct + let%test_unit "parse network created" = + let open Network_created in + let result = + {|{"network_id":"network0"}|} |> Yojson.Safe.from_string |> of_yojson + |> Result.ok_or_failwith + in + assert (equal result { network_id = "network0" }) + + (* TODO: fix *) + (* let%test_unit "parse network deployed response" = + let open Node_type in + let open Network_deployed in + let result = + {| + { "node0": ("Archive_node", "gql0") + , "node1": ("Block_producer_node", "gql1") + , "node2": ("Seed_node", "gql2") + , "node3": ("Snark_worker", null) + , "node4": ("Snark_coordinator", "gql4") + } + |} + |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith + in + assert ( + equal result + @@ of_alist_exn + [ ("node0", (Archive_node, Some "gql0")) + ; ("node1", (Block_producer_node, Some "gql1")) + ; ("node2", (Seed_node, Some "gql2")) + ; ("node3", (Snark_worker, None)) + ; ("node4", (Snark_coordinator, Some "gql4")) + ] ) *) + + let%test_unit "parse node started" = + let open Node_started in + let result = + {|{"node_id":"node0"}|} |> Yojson.Safe.from_string |> of_yojson + |> Result.ok_or_failwith + in + assert (equal result { node_id = "node0" }) + + let%test_unit "parse node stopped" = + let open Node_stopped in + let result = + {|{"node_id":"node0"}|} |> Yojson.Safe.from_string |> of_yojson + |> Result.ok_or_failwith + in + assert (equal result { node_id = "node0" }) + + let%test_unit "parse archive data dump" = + let open Archive_data_dump in + let result = + {|{"node_id":"node0","data":"data0"}|} |> Yojson.Safe.from_string + |> of_yojson |> Result.ok_or_failwith + in + assert (equal result { node_id = "node0"; data = "data0" }) + + let%test_unit "parse mina logs dump" = + let open Mina_logs_dump in + let logs = "{\"log0\":\"msg0\"}\n{\"log1\":\"msg1\"}" in + let result = + `Assoc [ ("node_id", `String "node0"); ("logs", `String logs) ] + |> of_yojson |> Result.ok_or_failwith + in + assert (equal result { node_id = "node0"; logs }) + + let%test_unit "parse precomputed block dump response" = + let open Precomputed_block_dump in + let result = + {|{"node_id":"node0","blocks":"blocks"}|} |> Yojson.Safe.from_string + |> of_yojson |> Result.ok_or_failwith + in + assert (equal result { node_id = "node0"; blocks = "blocks" }) + + let%test_unit "Replayer run response" = + let open Replayer_run in + let logs = "{\"log0\":\"msg0\"}\n{\"log1\":\"msg1\"}" in + let result = + `Assoc [ ("node_id", `String "node0"); ("logs", `String logs) ] + |> of_yojson |> Result.ok_or_failwith + in + assert (equal result { node_id = "node0"; logs }) +end + +let () = ignore @@ Async.Scheduler.go () diff --git a/src/lib/integration_test_lib/network_keypair.ml b/src/lib/integration_test_lib/network_keypair.ml index 4a46f7e432a..6800e30e123 100644 --- a/src/lib/integration_test_lib/network_keypair.ml +++ b/src/lib/integration_test_lib/network_keypair.ml @@ -8,7 +8,7 @@ type t = ; public_key : string ; private_key : string } -[@@deriving to_yojson] +[@@deriving eq, to_yojson] let create_network_keypair ~keypair_name ~keypair = let open Keypair in diff --git a/src/lib/integration_test_lib/util.ml b/src/lib/integration_test_lib/util.ml index deef6d87f66..341ee4eb8e0 100644 --- a/src/lib/integration_test_lib/util.ml +++ b/src/lib/integration_test_lib/util.ml @@ -9,6 +9,20 @@ let run_cmd dir prog args = Process.create_exn ~working_dir:dir ~prog ~args () >>= Process.collect_output_and_wait +let drop_outer_quotes s = + let open String in + let s = strip s in + let s_list = to_list s in + let f c = Char.equal c '"' in + let n = + Int.min + (List.take_while s_list ~f |> List.length) + (List.rev s_list |> List.take_while ~f |> List.length) + in + let s = drop_prefix s n in + let s = drop_suffix s n in + s + let check_cmd_output ~prog ~args output = let open Process.Output in let print_output () = @@ -31,7 +45,7 @@ let check_cmd_output ~prog ~args output = in match output.exit_status with | Ok () -> - return (Ok output.stdout) + return @@ Ok (drop_outer_quotes output.stdout) | Error (`Exit_non_zero status) -> let%map () = print_output () in Or_error.errorf "command exited with status code %d" status @@ -40,10 +54,12 @@ let check_cmd_output ~prog ~args output = Or_error.errorf "command exited prematurely due to signal %d" (Signal.to_system_int signal) -let run_cmd_or_error_timeout ~timeout_seconds dir prog args = - [%log' spam (Logger.create ())] - "Running command (from %s): $command" dir - ~metadata:[ ("command", `String (String.concat (prog :: args) ~sep:" ")) ] ; +let run_cmd_or_error_timeout ?(suppress_logs = false) ~timeout_seconds dir prog + args = + if not suppress_logs then + [%log' spam (Logger.create ())] + "Running command (from %s): $command" dir + ~metadata:[ ("command", `String (String.concat (prog :: args) ~sep:" ")) ] ; let open Deferred.Let_syntax in let%bind process = Process.create_exn ~working_dir:dir ~prog ~args () in let%bind res = From 9d82a9b8e5ed7d2dfa0db8faee1571b523aee321 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 2 Aug 2023 03:31:06 +0000 Subject: [PATCH 21/90] (abstract) Continue removing platform-specific commands from engine --- .../abstract_network.ml | 262 ++++++-------- .../graphql_polling_log_engine.ml | 16 +- .../mina_automation.ml | 328 +++--------------- src/lib/integration_test_lib/intf.ml | 2 +- 4 files changed, 161 insertions(+), 447 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index 815a883ad85..d199f112a7d 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -7,26 +7,17 @@ open Ci_interaction (* exclude from bisect_ppx to avoid type error on GraphQL modules *) [@@@coverage exclude_file] -type config = - { testnet_name : string - ; cluster : string - ; namespace : string - ; graphql_enabled : bool - ; network_id : Network_id.t - ; ingress_uri : string - ; current_commit_sha : string - } - (* [config_path] is instantiated when command line args are parsed *) let config_path = ref "" module Node = struct type t = - { node_id : string + { node_id : Node_id.t + ; network_id : Network_id.t ; network_keypair : Network_keypair.t option - ; config : config ; node_type : Node_type.t ; password : string + ; graphql_uri : string } let id { node_id; _ } = node_id @@ -34,30 +25,27 @@ module Node = struct let network_keypair { network_keypair; _ } = network_keypair (* TODO: remove *) - let base_kube_args t = [ "--cluster"; t.cluster; "--namespace"; t.namespace ] - - (* TODO: remove *) - let get_logs_in_container ?container_id { node_id; config; _ } = - let container_id = Option.value container_id ~default:"" in - let%bind cwd = Unix.getcwd () in - Integration_test_lib.Util.run_cmd_or_hard_error ~exit_code:13 cwd "kubectl" - (base_kube_args config @ [ "logs"; "-c"; container_id; node_id ]) - - let start ?commit_sha ~fresh_state t : unit Malleable_error.t = + let[@warning "-27"] get_logs_in_container ?container_id { node_id; _ } = + (* let container_id = Option.value container_id ~default:"" in + let%bind cwd = Unix.getcwd () in + Integration_test_lib.Util.run_cmd_or_hard_error ~exit_code:13 cwd "kubectl" + (base_kube_args config @ [ "logs"; "-c"; container_id; node_id ]) *) + failwith "get_logs_in_container" + + let start ?(git_commit = Mina_version.commit_id_short) ~fresh_state t : + unit Malleable_error.t = try - let commit_sha = - Option.value commit_sha ~default:t.config.current_commit_sha - in let args = - [ ("node_id", `String (id t)) + [ ("node_id", `String t.node_id) ; ("fresh_state", `Bool fresh_state) - ; ("commit_sha", `String commit_sha) + ; ("git_commit", `String git_commit) ] in let%bind () = match%map run_command ~config:!config_path ~args "start_node" with - | Ok (Response.Node_started _) -> - () + | Ok output -> + Yojson.Safe.from_string output + |> Node_started.of_yojson |> Result.ok_or_failwith |> ignore | _ -> failwith "invalid node started response" in @@ -66,11 +54,12 @@ module Node = struct let stop t = try - let args = [ ("node_id", `String (id t)) ] in + let args = [ ("node_id", `String t.node_id) ] in let%bind () = match%map run_command ~config:!config_path ~args "stop_node" with - | Ok (Response.Node_stopped _) -> - () + | Ok output -> + Yojson.Safe.from_string output + |> Node_stopped.of_yojson |> Result.ok_or_failwith |> ignore | _ -> failwith "invalid node stopped response" in @@ -78,7 +67,7 @@ module Node = struct with Failure err -> Malleable_error.hard_error_string err let logger_metadata node = - [ ("namespace", `String node.config.namespace) + [ ("network_id", `String node.network_id) ; ("node_id", `String node.node_id) ] @@ -87,11 +76,8 @@ module Node = struct module Graphql = struct (* TODO: fix *) let ingress_uri node = - let host = - sprintf "%s.%s" node.config.testnet_name node.config.ingress_uri - in - let path = sprintf "/%s/graphql" node.node_id in - Uri.make ~scheme:"http" ~host ~path ~port:80 () + let path = sprintf "/%s/graphql" node.graphql_uri in + Uri.make ~scheme:"http" ~path ~port:80 () module Client = Graphql_lib.Client.Make (struct let preprocess_variables_string = Fn.id @@ -324,56 +310,50 @@ module Node = struct let exec_graphql_request ?(num_tries = 10) ?(retry_delay_sec = 30.0) ?(initial_delay_sec = 0.) ~logger ~node ~query_name query_obj = let open Deferred.Let_syntax in - if not node.config.graphql_enabled then - Deferred.Or_error.error_string - "graphql is not enabled (hint: set `requires_graphql = true` in the \ - test config)" - else - let uri = Graphql.ingress_uri node in - let metadata = - [ ("query", `String query_name) - ; ("uri", `String (Uri.to_string uri)) - ; ("init_delay", `Float initial_delay_sec) - ] - in - [%log info] - "Attempting to send GraphQL request \"$query\" to \"$uri\" after \ - $init_delay sec" - ~metadata ; - let rec retry n = - if n <= 0 then ( - [%log error] - "GraphQL request \"$query\" to \"$uri\" failed too many times" - ~metadata ; - Deferred.Or_error.errorf - "GraphQL \"%s\" to \"%s\" request failed too many times" query_name - (Uri.to_string uri) ) - else - match%bind Graphql.Client.query query_obj uri with - | Ok result -> - [%log info] "GraphQL request \"$query\" to \"$uri\" succeeded" - ~metadata ; - Deferred.Or_error.return result - | Error (`Failed_request err_string) -> - [%log warn] - "GraphQL request \"$query\" to \"$uri\" failed: \"$error\" \ - ($num_tries attempts left)" - ~metadata: - ( metadata - @ [ ("error", `String err_string) - ; ("num_tries", `Int (n - 1)) - ] ) ; - let%bind () = after (Time.Span.of_sec retry_delay_sec) in - retry (n - 1) - | Error (`Graphql_error err_string) -> - [%log error] - "GraphQL request \"$query\" to \"$uri\" returned an error: \ - \"$error\" (this is a graphql error so not retrying)" - ~metadata:(("error", `String err_string) :: metadata) ; - Deferred.Or_error.error_string err_string - in - let%bind () = after (Time.Span.of_sec initial_delay_sec) in - retry num_tries + let uri = Graphql.ingress_uri node in + let metadata = + [ ("query", `String query_name) + ; ("uri", `String (Uri.to_string uri)) + ; ("init_delay", `Float initial_delay_sec) + ] + in + [%log info] + "Attempting to send GraphQL request \"$query\" to \"$uri\" after \ + $init_delay sec" + ~metadata ; + let rec retry n = + if n <= 0 then ( + [%log error] + "GraphQL request \"$query\" to \"$uri\" failed too many times" + ~metadata ; + Deferred.Or_error.errorf + "GraphQL \"%s\" to \"%s\" request failed too many times" query_name + (Uri.to_string uri) ) + else + match%bind Graphql.Client.query query_obj uri with + | Ok result -> + [%log info] "GraphQL request \"$query\" to \"$uri\" succeeded" + ~metadata ; + Deferred.Or_error.return result + | Error (`Failed_request err_string) -> + [%log warn] + "GraphQL request \"$query\" to \"$uri\" failed: \"$error\" \ + ($num_tries attempts left)" + ~metadata: + ( metadata + @ [ ("error", `String err_string); ("num_tries", `Int (n - 1)) ] + ) ; + let%bind () = after (Time.Span.of_sec retry_delay_sec) in + retry (n - 1) + | Error (`Graphql_error err_string) -> + [%log error] + "GraphQL request \"$query\" to \"$uri\" returned an error: \ + \"$error\" (this is a graphql error so not retrying)" + ~metadata:(("error", `String err_string) :: metadata) ; + Deferred.Or_error.error_string err_string + in + let%bind () = after (Time.Span.of_sec initial_delay_sec) in + retry num_tries let get_peer_ids ~logger t = let open Deferred.Or_error.Let_syntax in @@ -745,9 +725,7 @@ module Node = struct ~(zkapp_commands : Mina_base.Zkapp_command.t list) = [%log info] "Sending zkapp transactions" ~metadata: - [ ("namespace", `String t.config.namespace) - ; ("pod_id", `String (id t)) - ] ; + [ ("network_id", `String t.network_id); ("node_id", `String t.node_id) ] ; let open Deferred.Or_error.Let_syntax in let zkapp_commands_json = List.map zkapp_commands ~f:(fun zkapp_command -> @@ -797,8 +775,8 @@ module Node = struct ~(pk : Signature_lib.Public_key.Compressed.t) = [%log info] "Retrieving zkapp_commands from transaction pool" ~metadata: - [ ("namespace", `String t.config.namespace) - ; ("pod_id", `String (id t)) + [ ("network_id", `String t.network_id) + ; ("node_id", `String t.node_id) ; ("pub_key", Signature_lib.Public_key.Compressed.to_yojson pk) ] ; let open Deferred.Or_error.Let_syntax in @@ -819,8 +797,8 @@ module Node = struct in [%log info] "Retrieved zkapp_commands from transaction pool" ~metadata: - [ ("namespace", `String t.config.namespace) - ; ("pod_id", `String (id t)) + [ ("network_id", `String t.network_id) + ; ("node_id", `String t.node_id) ; ( "transaction ids" , `List (List.map ~f:(fun t -> `String t) transaction_ids) ) ] ; @@ -1021,74 +999,81 @@ module Node = struct "Node is not currently capturing structured log messages" let dump_archive_data ~logger (t : t) ~data_file = + let args = [ ("node_id", `String t.node_id) ] in if Node_type.(equal t.node_type Archive_node) then - let args = [ ("node_id", `String (id t)) ] in try - let%bind data = + let%bind dump = match%map run_command ~config:!config_path ~args "dump_archive_data" with - | Ok (Response.Archive_data_dump (_, logs)) -> - logs - | _ -> - failwith "invalid archive dump data response" + | Ok output -> + Yojson.Safe.from_string output + |> Archive_data_dump.of_yojson |> Result.ok_or_failwith + | Error err -> + raise @@ Archive_data_dump.Invalid_response err in [%log info] "Dumping archive data to file %s" data_file ; Malleable_error.return @@ Out_channel.with_file data_file ~f:(fun out_ch -> - Out_channel.output_string out_ch data ) + Out_channel.output_string out_ch dump.data ) with Failure err -> Malleable_error.hard_error_string err else + let node_type = Node_type.to_string t.node_type in + [%log error] "Node $node_id cannot dump archive data as a $node_type" + ~metadata:(("node_type", `String node_type) :: args) ; Malleable_error.hard_error_string - @@ sprintf "Node %s of type %s cannot dump archive data" (id t) - (Node_type.to_string t.node_type) + @@ sprintf "Node %s of type %s cannot dump archive data" t.node_id + node_type let run_replayer ~logger (t : t) = [%log info] "Running replayer on archived data node: %s" t.node_id ; - let args = [ ("node_id", `String (id t)) ] in + let args = [ ("node_id", `String t.node_id) ] in try - let%bind output = + let%bind replay = match%map run_command ~config:!config_path ~args "run_replayer" with - | Ok (Response.Replayer_run (_, output)) -> - output - | _ -> - failwith "invalid run replayer response" + | Ok output -> + Yojson.Safe.from_string output + |> Replayer_run.of_yojson |> Result.ok_or_failwith + | Error err -> + raise @@ Replayer_run.Invalid_response err in - Malleable_error.return output + Malleable_error.return replay.logs with Failure err -> Malleable_error.hard_error_string err let dump_mina_logs ~logger (t : t) ~log_file = [%log info] "Dumping logs from node: %s" t.node_id ; - let args = [ ("node_id", `String (id t)) ] in + let args = [ ("node_id", `String t.node_id) ] in try - let%bind logs = + let%bind dump = match%map run_command ~config:!config_path ~args "dump_mina_logs" with - | Ok (Response.Mina_logs_dump (_, logs)) -> - logs - | _ -> - failwith "invalid dump mina logs response" + | Ok output -> + Yojson.Safe.from_string output + |> Mina_logs_dump.of_yojson |> Result.ok_or_failwith + | Error err -> + raise @@ Mina_logs_dump.Invalid_response err in [%log info] "Dumping logs to file %s" log_file ; Malleable_error.return @@ Out_channel.with_file log_file ~f:(fun out_ch -> - Out_channel.output_string out_ch logs ) + Out_channel.output_string out_ch dump.logs ) with Failure err -> Malleable_error.hard_error_string err let dump_precomputed_blocks ~logger (t : t) = [%log info] "Dumping precomputed blocks from logs for node: %s" t.node_id ; - let args = [ ("node_id", `String (id t)) ] in + let args = [ ("node_id", `String t.node_id) ] in try - let%bind logs = + let%bind dump = match%map run_command ~config:!config_path ~args "dump_precomputed_blocks" with - | Ok (Response.Precomputed_block_dump (_, blocks)) -> - blocks - | _ -> - failwith "invalid dump precomputed blocks response" + | Ok output -> + Yojson.Safe.from_string output + |> Precomputed_block_dump.of_yojson |> Result.ok_or_failwith + | Error err -> + raise @@ Precomputed_block_dump.Invalid_response err in let log_lines = - String.split logs ~on:'\n' + String.split dump.blocks ~on:'\n' |> List.filter ~f:(String.is_prefix ~prefix:"{\"timestamp\":") in let jsons = List.map log_lines ~f:Yojson.Safe.from_string in @@ -1185,33 +1170,6 @@ module Node = struct } end -(* TODO: remove *) -module Workload_to_deploy = struct - type t = { workload_id : string; pod_info : string } - - (* TODO: remove *) - let base_kube_args { cluster; namespace; _ } = - [ "--cluster"; cluster; "--namespace"; namespace ] - - let construct_workload workload_id pod_info : t = { workload_id; pod_info } - - let[@warning "-27"] cons_pod_info ?network_keypair - ?(has_archive_container = false) primary_container_id = - "" - - let[@warning "-27"] get_nodes_from_workload t ~config = - let node_id = "node_id" in - (* TODO: get from ci runner *) - let node_type = Node_type.Seed_node in - (* TODO: get from ci runner *) - let network_keypair = None in - (* TODO: get from ci runner *) - let password = "password" in - (* TODO: get from ci runner *) - Malleable_error.return - { Node.node_id; node_type; config; network_keypair; password } -end - type t = { constants : Test_config.constants ; testnet_log_filter : string diff --git a/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml b/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml index a2d2a402cfb..4e52efc6793 100644 --- a/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml +++ b/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml @@ -50,7 +50,7 @@ let parse_event_from_log_entry ~logger log_entry = Event_type.parse_daemon_event msg | None -> (* Currently unreachable, but we could include error logs here if - desired. + desired. *) Event_type.parse_error_log msg in @@ -61,9 +61,7 @@ let rec poll_get_filtered_log_entries_node ~logger ~event_writer let open Deferred.Let_syntax in if not (Pipe.is_closed event_writer) then ( let%bind () = after (Time.Span.of_ms 10000.0) in - match%bind - Abstract_network.Node.get_filtered_log_entries ~last_log_index_seen node - with + match%bind Node.get_filtered_log_entries ~last_log_index_seen node with | Ok log_entries -> Array.iter log_entries ~f:(fun log_entry -> match parse_event_from_log_entry ~logger log_entry with @@ -80,7 +78,7 @@ let rec poll_get_filtered_log_entries_node ~logger ~event_writer | Error err -> [%log error] "Encountered an error while polling $node for logs: $err" ~metadata: - [ ("node", `String node.app_id) + [ ("node", `String node.node_id) ; ("err", Error_json.error_to_yojson err) ] ; (* Declare the node to be offline. *) @@ -93,9 +91,7 @@ let rec poll_get_filtered_log_entries_node ~logger ~event_writer let rec poll_start_filtered_log_node ~logger ~log_filter ~event_writer node = let open Deferred.Let_syntax in if not (Pipe.is_closed event_writer) then - match%bind - Abstract_network.Node.start_filtered_log ~logger ~log_filter node - with + match%bind Node.start_filtered_log ~logger ~log_filter node with | Ok () -> return (Ok ()) | Error _ -> @@ -106,12 +102,12 @@ let rec poll_node_for_logs_in_background ~log_filter ~logger ~event_writer (node : Node.t) = let open Deferred.Or_error.Let_syntax in [%log info] "Requesting for $node to start its filtered logs" - ~metadata:[ ("node", `String node.app_id) ] ; + ~metadata:[ ("node", `String node.node_id) ] ; let%bind () = poll_start_filtered_log_node ~logger ~log_filter ~event_writer node in [%log info] "$node has started its filtered logs. Beginning polling" - ~metadata:[ ("node", `String node.app_id) ] ; + ~metadata:[ ("node", `String node.node_id) ] ; let%bind () = poll_get_filtered_log_entries_node ~last_log_index_seen:0 ~logger ~event_writer node diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index 89eed4e6817..c5f0ac7ac62 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -6,23 +6,11 @@ open Ci_interaction module Network_manager = struct type t = { logger : Logger.t - ; testnet_name : string - ; cluster : string - ; namespace : string + ; network_id : string ; graphql_enabled : bool ; testnet_dir : string ; testnet_log_filter : string ; constants : Test_config.constants - ; seed_workloads : Abstract_network.Workload_to_deploy.t Core.String.Map.t - ; block_producer_workloads : - Abstract_network.Workload_to_deploy.t Core.String.Map.t - ; snark_coordinator_workloads : - Abstract_network.Workload_to_deploy.t Core.String.Map.t - ; snark_worker_workloads : - Abstract_network.Workload_to_deploy.t Core.String.Map.t - ; archive_workloads : - Abstract_network.Workload_to_deploy.t Core.String.Map.t - ; workloads_by_id : Abstract_network.Workload_to_deploy.t Core.String.Map.t ; mutable deployed : bool ; genesis_keypairs : Network_keypair.t Core.String.Map.t } @@ -34,104 +22,10 @@ module Network_manager = struct let run_cmd_or_hard_error t prog args = Util.run_cmd_or_hard_error t.testnet_dir prog args - let rec check_kube_capacity t ~logger ~(retries : int) ~(delay : float) : - unit Malleable_error.t = - let open Malleable_error.Let_syntax in - let%bind () = - Malleable_error.return ([%log info] "Running capacity check") - in - let%bind kubectl_top_nodes_output = - Util.run_cmd_or_hard_error "/" "kubectl" - [ "top"; "nodes"; "--sort-by=cpu"; "--no-headers" ] - in - let num_kube_nodes = - String.split_on_chars kubectl_top_nodes_output ~on:[ '\n' ] |> List.length - in - let%bind gcloud_descr_output = - Util.run_cmd_or_hard_error "/" "gcloud" - [ "container" - ; "clusters" - ; "describe" - ; cluster_name - ; "--project" - ; "o1labs-192920" - ; "--region" - ; cluster_region - ] - in - (* gcloud container clusters describe mina-integration-west1 --project o1labs-192920 --region us-west1 - this command gives us lots of information, including the max number of nodes per node pool. - *) - let%bind max_node_count_str = - Util.run_cmd_or_hard_error "/" "bash" - [ "-c" - ; Format.sprintf "echo \"%s\" | grep \"maxNodeCount\" " - gcloud_descr_output - ] - in - let max_node_count_by_node_pool = - Re2.find_all_exn (Re2.of_string "[0-9]+") max_node_count_str - |> List.map ~f:(fun str -> Int.of_string str) - in - (* We can have any number of node_pools. this string parsing will yield a list of ints, each int represents the - max_node_count for each node pool *) - let max_nodes = - List.fold max_node_count_by_node_pool ~init:0 ~f:(fun accum max_nodes -> - accum + (max_nodes * 3) ) - (* - the max_node_count_by_node_pool is per zone. us-west1 has 3 zones (we assume this never changes). - therefore to get the actual number of nodes a node_pool has, we multiply by 3. - then we sum up the number of nodes in all our node_pools to get the actual total maximum number of nodes that we can scale up to *) - in - let nodes_available = max_nodes - num_kube_nodes in - let cpus_needed_estimate = - 6 - * ( Core.Map.length t.seed_workloads - + Core.Map.length t.block_producer_workloads - + Core.Map.length t.snark_coordinator_workloads ) - (* as of 2022/07, the seed, bps, and the snark coordinator use 6 cpus. this is just a rough heuristic so we're not bothering to calculate memory needed *) - in - let cluster_nodes_needed = - Int.of_float - (Float.round_up (Float.( / ) (Float.of_int cpus_needed_estimate) 64.0)) - (* assuming that each node on the cluster has 64 cpus, as we've configured it to be in GCP as of *) - in - if nodes_available >= cluster_nodes_needed then - let%bind () = - Malleable_error.return - ([%log info] - "Capacity check passed. %d nodes are provisioned, the cluster \ - can scale up to a max of %d nodes. This test needs at least 1 \ - node to be unprovisioned." - num_kube_nodes max_nodes ) - in - Malleable_error.return () - else if retries <= 0 then - let%bind () = - Malleable_error.return - ([%log info] - "Capacity check failed. %d nodes are provisioned, the cluster \ - can scale up to a max of %d nodes. This test needs at least 1 \ - node to be unprovisioned. no more retries, thus exiting" - num_kube_nodes max_nodes ) - in - exit 7 - else - let%bind () = - Malleable_error.return - ([%log info] - "Capacity check failed. %d nodes are provisioned, the cluster \ - can scale up to a max of %d nodes. This test needs at least 1 \ - node to be unprovisioned. sleeping for 60 seconds before \ - retrying. will retry %d more times" - num_kube_nodes max_nodes (retries - 1) ) - in - let%bind () = Malleable_error.return (Thread.delay delay) in - check_kube_capacity t ~logger ~retries:(retries - 1) ~delay - let create ~logger (network_config : Network_config.t) = let open Malleable_error.Let_syntax in let%bind current_cluster = + (* TODO: replace with run_command *) Util.run_cmd_or_hard_error "/" "kubectl" [ "config"; "current-context" ] in [%log info] "Using cluster: %s" current_cluster ; @@ -142,7 +36,7 @@ module Network_manager = struct let all_namespaces = String.split ~on:' ' all_namespaces_str in let%bind () = if - List.mem all_namespaces network_config.terraform.testnet_name + List.mem all_namespaces network_config.config.network_id ~equal:String.equal then let%bind () = @@ -154,12 +48,13 @@ module Network_manager = struct start clean, and run the test; press Cntrl-C to quit out: " ) else Malleable_error.return - ([%log info] + @@ [%log info] "Existing namespace of same name detected; removing to start \ - clean" ) + clean" in + (* TODO: replace with run_command *) Util.run_cmd_or_hard_error "/" "kubectl" - [ "delete"; "namespace"; network_config.terraform.testnet_name ] + [ "delete"; "namespace"; network_config.config.network_id ] >>| Fn.const () else return () in @@ -167,138 +62,30 @@ module Network_manager = struct let testnet_log_filter = Network_config.testnet_log_filter network_config in (* we currently only deploy 1 seed and coordinator per deploy (will be configurable later) *) (* seed node keyname and workload name hardcoded as "seed" *) - let seed_workloads = - Core.String.Map.add_exn Core.String.Map.empty ~key:"seed" - ~data: - (Abstract_network.Workload_to_deploy.construct_workload "seed" - (Abstract_network.Workload_to_deploy.cons_pod_info "mina") ) - in - - let snark_coordinator_workloads, snark_worker_workloads = - match network_config.terraform.snark_coordinator_config with - | Some config -> - let snark_coordinator_workloads = - if config.worker_nodes > 0 then - Core.String.Map.of_alist_exn - [ ( config.name - , Abstract_network.Workload_to_deploy.construct_workload - config.name - (Abstract_network.Workload_to_deploy.cons_pod_info "mina") - ) - ] - else Core.String.Map.of_alist_exn [] - in - let snark_worker_workloads = - if config.worker_nodes > 0 then - Core.String.Map.of_alist_exn - [ ( config.name ^ "-worker" - , Abstract_network.Workload_to_deploy.construct_workload - (config.name ^ "-worker") - (Abstract_network.Workload_to_deploy.cons_pod_info - "worker" ) ) - ] - else Core.String.Map.of_alist_exn [] - in - (snark_coordinator_workloads, snark_worker_workloads) - | None -> - (Core.String.Map.of_alist_exn [], Core.String.Map.of_alist_exn []) - in - (* - let snark_coordinator_id = - String.lowercase - (String.sub network_config.terraform.snark_worker_public_key - ~pos: - (String.length network_config.terraform.snark_worker_public_key - 6) - ~len:6 ) - in - let snark_coordinator_workloads = - if network_config.terraform.snark_worker_replicas > 0 then - [ Abstract_network.Workload_to_deploy.construct_workload - ("snark-coordinator-" ^ snark_coordinator_id) - [ Abstract_network.Workload_to_deploy.cons_pod_info "mina" ] - ] - else [] - in - let snark_worker_workloads = - if network_config.terraform.snark_worker_replicas > 0 then - [ Abstract_network.Workload_to_deploy.construct_workload - ("snark-worker-" ^ snark_coordinator_id) - (List.init network_config.terraform.snark_worker_replicas - ~f:(fun _i -> - Abstract_network.Workload_to_deploy.cons_pod_info "worker" ) - ) - ] - else [] - in *) - let block_producer_workloads = - List.map network_config.terraform.block_producer_configs - ~f:(fun bp_config -> - ( bp_config.name - , Abstract_network.Workload_to_deploy.construct_workload - bp_config.name - (Abstract_network.Workload_to_deploy.cons_pod_info - ~network_keypair:bp_config.keypair "mina" ) ) ) - |> Core.String.Map.of_alist_exn - in - let archive_workloads = - List.init network_config.terraform.archive_node_count ~f:(fun i -> - ( sprintf "archive-%d" (i + 1) - , Abstract_network.Workload_to_deploy.construct_workload - (sprintf "archive-%d" (i + 1)) - (Abstract_network.Workload_to_deploy.cons_pod_info - ~has_archive_container:true "mina" ) ) ) - |> Core.String.Map.of_alist_exn - in - let workloads_by_id = - let all_workloads = - Core.String.Map.data seed_workloads - @ Core.String.Map.data snark_coordinator_workloads - @ Core.String.Map.data snark_worker_workloads - @ Core.String.Map.data block_producer_workloads - @ Core.String.Map.data archive_workloads - in - all_workloads - |> List.map ~f:(fun w -> (w.workload_id, w)) - |> String.Map.of_alist_exn - in let testnet_dir = - network_config.mina_automation_location ^/ "terraform/testnets" - ^/ network_config.terraform.testnet_name + network_config.config.config_dir ^/ "/testnets" + ^/ network_config.config.network_id in let t = { logger - ; cluster = cluster_id - ; namespace = network_config.terraform.testnet_name - ; testnet_name = network_config.terraform.testnet_name - ; graphql_enabled = network_config.terraform.deploy_graphql_ingress + ; network_id = network_config.config.network_id + ; graphql_enabled = true ; testnet_dir ; testnet_log_filter ; constants = network_config.constants - ; seed_workloads - ; block_producer_workloads - ; snark_coordinator_workloads - ; snark_worker_workloads - ; archive_workloads - ; workloads_by_id ; deployed = false ; genesis_keypairs = network_config.genesis_keypairs } in - (* making the main.tf.json *) let open Deferred.Let_syntax in - let%bind () = - if%bind File_system.dir_exists testnet_dir then ( - [%log info] "Old terraform directory found; removing to start clean" ; - File_system.remove_dir testnet_dir ) - else return () - in [%log info] "Making testnet dir %s" testnet_dir ; let%bind () = Unix.mkdir testnet_dir in - let tf_filename = testnet_dir ^/ "main.tf.json" in - [%log info] "Writing network configuration into %s" tf_filename ; - Out_channel.with_file ~fail_if_exists:true tf_filename ~f:(fun ch -> - Network_config.to_terraform network_config - |> Terraform.to_string + let network_config_filename = testnet_dir ^/ "network_config.json" in + [%log info] "Writing network configuration into %s" network_config_filename ; + Out_channel.with_file ~fail_if_exists:true network_config_filename + ~f:(fun ch -> + Network_config.to_yojson network_config + |> Yojson.Safe.to_string |> Out_channel.output_string ch ) ; [%log info] "Writing out the genesis keys (in case you want to use them manually) to \ @@ -313,70 +100,43 @@ module Network_manager = struct (String.concat [ kps_base_path; "/"; kp.keypair_name; ".json" ]) ) |> Deferred.return in - [%log info] "Initializing terraform" ; - let open Malleable_error.Let_syntax in - let%bind (_ : string) = run_cmd_or_hard_error t "terraform" [ "init" ] in - let%map (_ : string) = run_cmd_or_hard_error t "terraform" [ "validate" ] in - t + Malleable_error.return t + (* TODO: use output *) let deploy t = - let open Malleable_error.Let_syntax in let logger = t.logger in if t.deployed then failwith "network already deployed" ; [%log info] "Deploying network" ; - let%bind (_ : string) = - run_cmd_or_hard_error t "terraform" [ "apply"; "-auto-approve" ] - in + let%bind output = + let open Network_deployed in + match%map + run_command + ~config:!Abstract_network.config_path + ~args:[] "deploy_network" + with + | Ok output -> + output |> Yojson.Safe.from_string |> of_yojson + |> Result.ok_or_failwith + | Error err -> + raise @@ Invalid_output err + in + let _ = Map.is_empty output in t.deployed <- true ; - let config : Abstract_network.config = - { testnet_name = t.testnet_name - ; cluster = t.cluster - ; namespace = t.namespace - ; graphql_enabled = t.graphql_enabled - ; access_token = "access_token" (* TODO: *) - ; network_id = "network_id" (* TODO: *) - ; ingress_uri = "ingress_uri" (* TODO: *) - ; current_commit_sha = "0000000" (* TODO: *) - } - in - let func_for_fold ~(key : string) ~data accum_M = - let%bind mp = accum_M in - let%map node = - Abstract_network.Workload_to_deploy.get_nodes_from_workload data ~config - in - Core.String.Map.add_exn mp ~key ~data:node - in - let%map seeds = - Core.String.Map.fold t.seed_workloads - ~init:(Malleable_error.return Core.String.Map.empty) - ~f:func_for_fold - and block_producers = - Core.String.Map.fold t.block_producer_workloads - ~init:(Malleable_error.return Core.String.Map.empty) - ~f:func_for_fold - and snark_coordinators = - Core.String.Map.fold t.snark_coordinator_workloads - ~init:(Malleable_error.return Core.String.Map.empty) - ~f:func_for_fold - and snark_workers = - Core.String.Map.fold t.snark_worker_workloads - ~init:(Malleable_error.return Core.String.Map.empty) - ~f:func_for_fold - and archive_nodes = - Core.String.Map.fold t.archive_workloads - ~init:(Malleable_error.return Core.String.Map.empty) - ~f:func_for_fold - in + let seeds = Core.String.Map.empty in + let block_producers = Core.String.Map.empty in + let snark_coordinators = Core.String.Map.empty in + let snark_workers = Core.String.Map.empty in + let archive_nodes = Core.String.Map.empty in let network = - { Abstract_network.network_id = t.namespace (* TODO: fix *) - ; constants = t.constants + { Abstract_network.constants = t.constants + ; testnet_log_filter = t.testnet_log_filter + ; genesis_keypairs = t.genesis_keypairs ; seeds ; block_producers ; snark_coordinators ; snark_workers ; archive_nodes - ; testnet_log_filter = t.testnet_log_filter - ; genesis_keypairs = t.genesis_keypairs + ; network_id = t.network_id } in let nodes_to_string = @@ -384,7 +144,7 @@ module Network_manager = struct (List.map ~f:Abstract_network.Node.id) in [%log info] "Network deployed" ; - [%log info] "testnet namespace: %s" t.namespace ; + [%log info] "network id: %s" t.network_id ; [%log info] "snark coordinators: %s" (nodes_to_string (Core.String.Map.data network.snark_coordinators)) ; [%log info] "snark workers: %s" @@ -393,7 +153,7 @@ module Network_manager = struct (nodes_to_string (Core.String.Map.data network.block_producers)) ; [%log info] "archive nodes: %s" (nodes_to_string (Core.String.Map.data network.archive_nodes)) ; - network + Malleable_error.return network let destroy t = [%log' info t.logger] "Destroying network" ; diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index 5982f2e7008..236d76d1a6b 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -52,7 +52,7 @@ module Engine = struct val network_keypair : t -> Network_keypair.t option val start : - ?commit_sha:string -> fresh_state:bool -> t -> unit Malleable_error.t + ?git_commit:string -> fresh_state:bool -> t -> unit Malleable_error.t val stop : t -> unit Malleable_error.t From a14b7da8da978f59400381b225f985a776310c79 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 2 Aug 2023 03:32:03 +0000 Subject: [PATCH 22/90] (test exec) Allow use of GCP engine --- src/app/test_executive/dune | 2 ++ .../integration_test_cloud_engine/kubernetes_network.ml | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/app/test_executive/dune b/src/app/test_executive/dune index b637a93c718..37e12a8ce17 100644 --- a/src/app/test_executive/dune +++ b/src/app/test_executive/dune @@ -14,6 +14,7 @@ async_unix unsigned_extended stdio + uri ;; local libraries mina_wire_types with_hash @@ -37,6 +38,7 @@ mina_runtime_config secrets integration_test_abstract_engine + integration_test_cloud_engine mina_generators logger random_oracle diff --git a/src/lib/integration_test_cloud_engine/kubernetes_network.ml b/src/lib/integration_test_cloud_engine/kubernetes_network.ml index b536cacf7b3..1d5feb2c805 100644 --- a/src/lib/integration_test_cloud_engine/kubernetes_network.ml +++ b/src/lib/integration_test_cloud_engine/kubernetes_network.ml @@ -5,6 +5,10 @@ open Integration_test_lib (* exclude from bisect_ppx to avoid type error on GraphQL modules *) [@@@coverage exclude_file] +let config_path = ref "N/A" + +let id _ = "cloud" + let mina_archive_container_id = "archive" let mina_archive_username = "mina" @@ -105,7 +109,8 @@ module Node = struct Integration_test_lib.Util.run_cmd_or_error cwd "kubectl" (base_kube_args config @ [ "cp"; "-c"; container_id; tmp_file; dest_file ]) - let start ~fresh_state node : unit Malleable_error.t = + let[@warning "-27"] start ?(git_commit = "") ~fresh_state node : + unit Malleable_error.t = let open Malleable_error.Let_syntax in node.should_be_running <- true ; let%bind () = From c03f30d41a7c201d5ff8108f08fb43834b1b2225 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 2 Aug 2023 20:26:30 +0000 Subject: [PATCH 23/90] (config) Update network deploy response parsing and unit test --- .../ci_interaction.ml | 68 +++---- .../unit_tests.ml | 166 ++++++++++++++---- .../integration_test_lib/network_keypair.ml | 2 +- 3 files changed, 172 insertions(+), 64 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 52d27282950..4e6b93a2362 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -406,57 +406,65 @@ module Network_deployed = struct exception Invalid_keypair of Yojson.Safe.t - type t = node_info Map.t [@@deriving eq] - - and node_info = + type node_info = { node_type : Node_type.t ; network_keypair : Network_keypair.t option - ; password : string - ; graphql_uri : string + ; graphql_uri : string option } - [@@deriving yojson] + [@@deriving to_yojson] + + let equal_node_info m n = + let node_type = Node_type.equal m.node_type n.node_type in + let network_keypair = + Option.equal + (fun (m : Network_keypair.t) n -> + Public_key.equal m.keypair.public_key n.keypair.public_key + && String.equal m.public_key n.public_key + && String.equal m.privkey_password n.privkey_password + && String.equal m.keypair_name n.keypair_name ) + m.network_keypair n.network_keypair + in + let graphql = Option.equal String.equal m.graphql_uri n.graphql_uri in + node_type && network_keypair && graphql + + type t = node_info Map.t [@@deriving eq] - let network_keypair_of_yojson (sk : Yojson.Safe.t) = - match sk with + let network_keypair_of_yojson node_id (private_key : Yojson.Safe.t) = + match private_key with | `Null -> None - | `Assoc - [ ("private_key", `String private_key) - ; ("privkey_password", `String privkey_password) - ; ("keypair_name", `String keypair_name) - ] -> + | `String private_key -> + let keypair_name = node_id ^ "_key" in let keypair = Keypair.of_private_key_exn @@ Private_key.of_base58_check_exn private_key in - Some - Network_keypair. - { keypair - ; keypair_name - ; privkey_password - ; private_key - ; public_key = - Public_key.to_bigstring keypair.public_key - |> Bigstring.to_string - } + Some (Network_keypair.create_network_keypair ~keypair_name ~keypair) + | _ -> + raise @@ Invalid_keypair private_key + + let graphql_uri_of_yojson (uri : Yojson.Safe.t) = + match uri with + | `Null -> + None + | `String uri -> + Some uri | _ -> - raise @@ Invalid_keypair sk + raise @@ Invalid_argument (Yojson.Safe.to_string uri) let of_yojson = let f accum = function | ( node_id , `Assoc [ ("node_type", `String nt) - ; ("network_keypair", keypair) - ; ("password", `String password) - ; ("graphql_uri", `String graphql_uri) + ; ("private_key", private_key) + ; ("graphql_uri", graphql_uri) ] ) -> Map.set accum ~key:node_id ~data: { node_type = Node_type.of_string nt - ; network_keypair = network_keypair_of_yojson keypair - ; password - ; graphql_uri + ; network_keypair = network_keypair_of_yojson node_id private_key + ; graphql_uri = graphql_uri_of_yojson graphql_uri } | node_id, t -> raise @@ Invalid_entry (node_id, t) diff --git a/src/lib/integration_test_abstract_engine/unit_tests.ml b/src/lib/integration_test_abstract_engine/unit_tests.ml index b5651e95103..26585d5ed7f 100644 --- a/src/lib/integration_test_abstract_engine/unit_tests.ml +++ b/src/lib/integration_test_abstract_engine/unit_tests.ml @@ -17,7 +17,7 @@ module Test_values = struct let start_node_raw_cmd = "minimina start-node --node-id {{node_id}} --fresh-state {{fresh_state}} \ - --commit-sha {{commit_sha}}" + --git-commit {{git_commit}}" let deploy_network_action = sprintf @@ -41,7 +41,7 @@ module Test_values = struct "args": { "node_id": "string", "fresh_state": "bool", - "commit_sha": "string" + "git_commit": "string" }, "command": "%s" } @@ -88,7 +88,7 @@ module Helper_tests = struct let args = [ ("node_id", `String "node0") ; ("fresh_state", `Bool true) - ; ("commit_sha", `String "0123456abcdef") + ; ("git_commit", `String "0123456abcdef") ] in assert (validate_args ~args ~action) @@ -183,8 +183,6 @@ module Run_command_tests = struct end module Request_tests = struct - let ( = ) = String.equal - let%test_unit "Create network request" = let constants : Test_config.constants = { constraints = Genesis_constants.Constraint_constants.compiled @@ -194,12 +192,47 @@ module Request_tests = struct let network_config : Network_config.t' = { debug_arg = true; genesis_keypairs = Core.String.Map.empty; constants } in - let result = - Network_config.t'_to_yojson network_config |> Yojson.Safe.to_string + let expect = + {| + { "debug_arg":true, + "genesis_keypairs":{}, + "constants":{ + "constraints":{ + "sub_windows_per_window":11, + "ledger_depth":35, + "work_delay":2, + "block_window_duration_ms":180000, + "transaction_capacity_log_2":7, + "pending_coinbase_depth":5, + "coinbase_amount":"720000000000", + "supercharged_coinbase_factor":1, + "account_creation_fee":"1", + "fork":null + }, + "genesis":{ + "protocol":{ + "k":290, + "slots_per_epoch":7140, + "slots_per_sub_window":7, + "delta":0, + "genesis_state_timestamp":"2020-09-16 10:15:00.000000Z" + }, + "txpool_max_size":3000, + "num_accounts":null, + "zkapp_proof_update_cost":10.26, + "zkapp_signed_single_update_cost":9.140000000000001, + "zkapp_signed_pair_update_cost":10.08, + "zkapp_transaction_cost_limit":69.45, + "max_event_elements":100, + "max_action_elements":100 + } + } + } + |} + |> Yojson.Safe.from_string in assert ( - result - = {|{"debug_arg":true,"genesis_keypairs":{},"constants":{"constraints":{"sub_windows_per_window":11,"ledger_depth":35,"work_delay":2,"block_window_duration_ms":180000,"transaction_capacity_log_2":7,"pending_coinbase_depth":5,"coinbase_amount":"720000000000","supercharged_coinbase_factor":1,"account_creation_fee":"1","fork":null},"genesis":{"protocol":{"k":290,"slots_per_epoch":7140,"slots_per_sub_window":7,"delta":0,"genesis_state_timestamp":"2020-09-16 10:15:00.000000Z"},"txpool_max_size":3000,"num_accounts":null,"zkapp_proof_update_cost":10.26,"zkapp_signed_single_update_cost":9.140000000000001,"zkapp_signed_pair_update_cost":10.08,"zkapp_transaction_cost_limit":69.45,"max_event_elements":100,"max_action_elements":100}}}|} ) + Yojson.Safe.equal expect @@ Network_config.t'_to_yojson network_config ) end module Parse_output_tests = struct @@ -211,30 +244,97 @@ module Parse_output_tests = struct in assert (equal result { network_id = "network0" }) - (* TODO: fix *) - (* let%test_unit "parse network deployed response" = - let open Node_type in - let open Network_deployed in - let result = - {| - { "node0": ("Archive_node", "gql0") - , "node1": ("Block_producer_node", "gql1") - , "node2": ("Seed_node", "gql2") - , "node3": ("Snark_worker", null) - , "node4": ("Snark_coordinator", "gql4") - } - |} - |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith - in - assert ( - equal result - @@ of_alist_exn - [ ("node0", (Archive_node, Some "gql0")) - ; ("node1", (Block_producer_node, Some "gql1")) - ; ("node2", (Seed_node, Some "gql2")) - ; ("node3", (Snark_worker, None)) - ; ("node4", (Snark_coordinator, Some "gql4")) - ] ) *) + let%test_unit "parse network deployed response" = + let open Node_type in + let open Network_deployed in + let result = + {| + { "node0": { + "node_type":"Archive_node", + "private_key":null, + "graphql_uri":"gql_archive" + }, + "node1": { + "node_type":"Block_producer_node", + "private_key":null, + "graphql_uri":"gql_bp" + }, + "node2": { + "node_type":"Seed_node", + "private_key":"EKEQpDAjj7dP3j7fQy4qBU7Kxns85wwq5xMn4zxdyQm83pEWzQ62", + "graphql_uri":"gql_seed" + }, + "node3": { + "node_type":"Snark_worker", + "private_key":null, + "graphql_uri":"gql_snark" + }, + "node4": { + "node_type":"Snark_coordinator", + "private_key":null, + "graphql_uri":null + } + } + |} + |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith + in + let seed_keypair = + let open Signature_lib in + let keypair = + let private_key = + Private_key.of_base58_check_exn + "EKEQpDAjj7dP3j7fQy4qBU7Kxns85wwq5xMn4zxdyQm83pEWzQ62" + in + let public_key = Public_key.of_private_key_exn private_key in + Keypair.{ public_key; private_key } + in + Network_keypair.create_network_keypair ~keypair_name:"node2_key" ~keypair + in + let archive = + { node_type = Archive_node + ; network_keypair = None + ; graphql_uri = Some "gql_archive" + } + in + let bp = + { node_type = Block_producer_node + ; network_keypair = None + ; graphql_uri = Some "gql_bp" + } + in + let seed = + { node_type = Seed_node + ; network_keypair = Some seed_keypair + ; graphql_uri = Some "gql_seed" + } + in + let worker = + { node_type = Snark_worker + ; network_keypair = None + ; graphql_uri = Some "gql_snark" + } + in + let coordinator = + { node_type = Snark_coordinator + ; network_keypair = None + ; graphql_uri = None + } + in + let expect = + of_alist_exn + [ ("node0", archive) + ; ("node1", bp) + ; ("node2", seed) + ; ("node3", worker) + ; ("node4", coordinator) + ] + in + assert ( + (* print_endline "=== Result ===" ; + Map.iter result ~f:(fun ni -> node_info_to_yojson ni |> Yojson.Safe.to_string |> print_endline) ; + print_endline "\n\n=== Expect ===" ; + Map.iter expect ~f:(fun ni -> node_info_to_yojson ni |> Yojson.Safe.to_string |> print_endline) ; *) + equal result expect ) let%test_unit "parse node started" = let open Node_started in diff --git a/src/lib/integration_test_lib/network_keypair.ml b/src/lib/integration_test_lib/network_keypair.ml index 6800e30e123..4a46f7e432a 100644 --- a/src/lib/integration_test_lib/network_keypair.ml +++ b/src/lib/integration_test_lib/network_keypair.ml @@ -8,7 +8,7 @@ type t = ; public_key : string ; private_key : string } -[@@deriving eq, to_yojson] +[@@deriving to_yojson] let create_network_keypair ~keypair_name ~keypair = let open Keypair in From 4699f6e6e224e809864b50a52a389f846c87b480 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Thu, 3 Aug 2023 00:24:12 +0000 Subject: [PATCH 24/90] (config) Refactor netowrk deployed --- .../abstract_network.ml | 148 +++++--- .../ci_interaction.ml | 327 +++++++++--------- .../config.json | 68 +++- .../mina_automation.ml | 48 ++- .../unit_tests.ml | 99 +++--- 5 files changed, 411 insertions(+), 279 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index d199f112a7d..4b8593e9694 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -36,13 +36,16 @@ module Node = struct unit Malleable_error.t = try let args = - [ ("node_id", `String t.node_id) + [ ("network_id", `String t.network_id) + ; ("node_id", `String t.node_id) ; ("fresh_state", `Bool fresh_state) ; ("git_commit", `String git_commit) ] in let%bind () = - match%map run_command ~config:!config_path ~args "start_node" with + match%map + Config_file.run_command ~config:!config_path ~args "start_node" + with | Ok output -> Yojson.Safe.from_string output |> Node_started.of_yojson |> Result.ok_or_failwith |> ignore @@ -54,9 +57,13 @@ module Node = struct let stop t = try - let args = [ ("node_id", `String t.node_id) ] in + let args = + [ ("network_id", `String t.network_id); ("node_id", `String t.node_id) ] + in let%bind () = - match%map run_command ~config:!config_path ~args "stop_node" with + match%map + Config_file.run_command ~config:!config_path ~args "stop_node" + with | Ok output -> Yojson.Safe.from_string output |> Node_stopped.of_yojson |> Result.ok_or_failwith |> ignore @@ -71,10 +78,39 @@ module Node = struct ; ("node_id", `String node.node_id) ] + module Collections = struct + let f network_id (node_info : Network_deployed.node_info) = + { node_id = node_info.node_id + ; network_id + ; network_keypair = node_info.network_keypair + ; node_type = node_info.node_type + ; password = "naughty blue worm" + ; graphql_uri = Option.value node_info.graphql_uri ~default:"" + } + + open Network_deployed + open Core.String.Map + + let network_id t = data t |> List.hd_exn |> network_id + + let archive_nodes t = + filter ~f:is_archive_node t |> map ~f:(f @@ network_id t) + + let block_producers t = + filter ~f:is_block_producer t |> map ~f:(f @@ network_id t) + + let seeds t = filter ~f:is_seed_node t |> map ~f:(f @@ network_id t) + + let snark_coordinators t = + filter t ~f:is_snark_coordinator |> map ~f:(f @@ network_id t) + + let snark_workers t = + filter ~f:is_snark_worker t |> map ~f:(f @@ network_id t) + end + module Scalars = Graphql_lib.Scalars module Graphql = struct - (* TODO: fix *) let ingress_uri node = let path = sprintf "/%s/graphql" node.graphql_uri in Uri.make ~scheme:"http" ~path ~port:80 () @@ -998,13 +1034,52 @@ module Node = struct Deferred.Or_error.error_string "Node is not currently capturing structured log messages" + let get_metrics ~logger t = + let open Deferred.Or_error.Let_syntax in + [%log info] "Getting node's metrics" ~metadata:(logger_metadata t) ; + let query_obj = Graphql.Query_metrics.make () in + let%bind query_result_obj = + exec_graphql_request ~logger ~node:t ~query_name:"query_metrics" query_obj + in + [%log info] "get_metrics, finished exec_graphql_request" ; + let block_production_delay = + Array.to_list + @@ query_result_obj.daemonStatus.metrics.blockProductionDelay + in + let metrics = query_result_obj.daemonStatus.metrics in + let transaction_pool_diff_received = metrics.transactionPoolDiffReceived in + let transaction_pool_diff_broadcasted = + metrics.transactionPoolDiffBroadcasted + in + let transactions_added_to_pool = metrics.transactionsAddedToPool in + let transaction_pool_size = metrics.transactionPoolSize in + [%log info] + "get_metrics, result of graphql query (block_production_delay; \ + tx_received; tx_broadcasted; txs_added_to_pool; tx_pool_size) (%s; %d; \ + %d; %d; %d)" + ( String.concat ~sep:", " + @@ List.map ~f:string_of_int block_production_delay ) + transaction_pool_diff_received transaction_pool_diff_broadcasted + transactions_added_to_pool transaction_pool_size ; + return + Intf. + { block_production_delay + ; transaction_pool_diff_broadcasted + ; transaction_pool_diff_received + ; transactions_added_to_pool + ; transaction_pool_size + } + let dump_archive_data ~logger (t : t) ~data_file = - let args = [ ("node_id", `String t.node_id) ] in + let args = + [ ("network_id", `String t.network_id); ("node_id", `String t.node_id) ] + in if Node_type.(equal t.node_type Archive_node) then try let%bind dump = match%map - run_command ~config:!config_path ~args "dump_archive_data" + Config_file.run_command ~config:!config_path ~args + "dump_archive_data" with | Ok output -> Yojson.Safe.from_string output @@ -1027,10 +1102,14 @@ module Node = struct let run_replayer ~logger (t : t) = [%log info] "Running replayer on archived data node: %s" t.node_id ; - let args = [ ("node_id", `String t.node_id) ] in + let args = + [ ("network_id", `String t.network_id); ("node_id", `String t.node_id) ] + in try let%bind replay = - match%map run_command ~config:!config_path ~args "run_replayer" with + match%map + Config_file.run_command ~config:!config_path ~args "run_replayer" + with | Ok output -> Yojson.Safe.from_string output |> Replayer_run.of_yojson |> Result.ok_or_failwith @@ -1042,10 +1121,14 @@ module Node = struct let dump_mina_logs ~logger (t : t) ~log_file = [%log info] "Dumping logs from node: %s" t.node_id ; - let args = [ ("node_id", `String t.node_id) ] in + let args = + [ ("network_id", `String t.network_id); ("node_id", `String t.node_id) ] + in try let%bind dump = - match%map run_command ~config:!config_path ~args "dump_mina_logs" with + match%map + Config_file.run_command ~config:!config_path ~args "dump_mina_logs" + with | Ok output -> Yojson.Safe.from_string output |> Mina_logs_dump.of_yojson |> Result.ok_or_failwith @@ -1060,11 +1143,14 @@ module Node = struct let dump_precomputed_blocks ~logger (t : t) = [%log info] "Dumping precomputed blocks from logs for node: %s" t.node_id ; - let args = [ ("node_id", `String t.node_id) ] in + let args = + [ ("network_id", `String t.network_id); ("node_id", `String t.node_id) ] + in try let%bind dump = match%map - run_command ~config:!config_path ~args "dump_precomputed_blocks" + Config_file.run_command ~config:!config_path ~args + "dump_precomputed_blocks" with | Ok output -> Yojson.Safe.from_string output @@ -1132,42 +1218,6 @@ module Node = struct in Malleable_error.return () with Failure err -> Malleable_error.hard_error_string err - - let get_metrics ~logger t = - let open Deferred.Or_error.Let_syntax in - [%log info] "Getting node's metrics" ~metadata:(logger_metadata t) ; - let query_obj = Graphql.Query_metrics.make () in - let%bind query_result_obj = - exec_graphql_request ~logger ~node:t ~query_name:"query_metrics" query_obj - in - [%log info] "get_metrics, finished exec_graphql_request" ; - let block_production_delay = - Array.to_list - @@ query_result_obj.daemonStatus.metrics.blockProductionDelay - in - let metrics = query_result_obj.daemonStatus.metrics in - let transaction_pool_diff_received = metrics.transactionPoolDiffReceived in - let transaction_pool_diff_broadcasted = - metrics.transactionPoolDiffBroadcasted - in - let transactions_added_to_pool = metrics.transactionsAddedToPool in - let transaction_pool_size = metrics.transactionPoolSize in - [%log info] - "get_metrics, result of graphql query (block_production_delay; \ - tx_received; tx_broadcasted; txs_added_to_pool; tx_pool_size) (%s; %d; \ - %d; %d; %d)" - ( String.concat ~sep:", " - @@ List.map ~f:string_of_int block_production_delay ) - transaction_pool_diff_received transaction_pool_diff_broadcasted - transactions_added_to_pool transaction_pool_size ; - return - Intf. - { block_production_delay - ; transaction_pool_diff_broadcasted - ; transaction_pool_diff_received - ; transactions_added_to_pool - ; transaction_pool_size - } end type t = diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 4e6b93a2362..7f7beb0d8c9 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -72,6 +72,7 @@ module Network_config = struct let testnet_log_filter t = t.config.network_id + (* TODO: double check *) let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) = @@ -330,23 +331,6 @@ module Network_config = struct } end -module Config_file = struct - type t = { version : int; actions : action list } [@@deriving eq, yojson] - - and action = { name : string; args : Yojson.Safe.t; command : string } - [@@deriving eq, yojson] - - exception Invalid_version of Yojson.Safe.t - - exception Invalid_actions of Yojson.Safe.t - - exception Invalid_args of Yojson.Safe.t - - exception Invalid_arg_type_should_be_string of string * Yojson.Safe.t - - exception Invalid_command of Yojson.Safe.t -end - module Network_id = struct type t = string [@@deriving eq, yojson] end @@ -397,17 +381,16 @@ module Network_created = struct end module Network_deployed = struct - module Map = Map.Make (String) - include Map - - exception Invalid_entry of string * Yojson.Safe.t + exception Invalid_entry of string exception Invalid_output of string exception Invalid_keypair of Yojson.Safe.t type node_info = - { node_type : Node_type.t + { node_id : Node_id.t + ; node_type : Node_type.t + ; network_id : Network_id.t ; network_keypair : Network_keypair.t option ; graphql_uri : string option } @@ -427,7 +410,21 @@ module Network_deployed = struct let graphql = Option.equal String.equal m.graphql_uri n.graphql_uri in node_type && network_keypair && graphql - type t = node_info Map.t [@@deriving eq] + let is_archive_node node_info = Node_type.is_archive_node node_info.node_type + + let is_block_producer node_info = + Node_type.is_block_producer node_info.node_type + + let is_seed_node node_info = Node_type.is_seed_node node_info.node_type + + let is_snark_coordinator node_info = + Node_type.is_snark_coordinator node_info.node_type + + let is_snark_worker node_info = Node_type.is_snark_worker node_info.node_type + + let network_id { network_id; _ } = network_id + + type t = node_info Core.String.Map.t [@@deriving eq] let network_keypair_of_yojson node_id (private_key : Yojson.Safe.t) = match private_key with @@ -453,25 +450,29 @@ module Network_deployed = struct raise @@ Invalid_argument (Yojson.Safe.to_string uri) let of_yojson = - let f accum = function - | ( node_id - , `Assoc - [ ("node_type", `String nt) - ; ("private_key", private_key) - ; ("graphql_uri", graphql_uri) - ] ) -> - Map.set accum ~key:node_id + let f network_id accum = function + | `Assoc + [ ( node_id + , `Assoc + [ ("node_type", `String nt) + ; ("private_key", private_key) + ; ("graphql_uri", graphql_uri) + ] ) + ] -> + Core.String.Map.set accum ~key:node_id ~data: - { node_type = Node_type.of_string nt + { node_id + ; network_id + ; node_type = Node_type.of_string nt ; network_keypair = network_keypair_of_yojson node_id private_key ; graphql_uri = graphql_uri_of_yojson graphql_uri } - | node_id, t -> - raise @@ Invalid_entry (node_id, t) + | t -> + raise @@ Invalid_entry (Yojson.Safe.to_string t) in function - | `Assoc a_list -> - Ok (List.fold a_list ~init:(Map.empty : t) ~f) + | `Assoc [ ("network_id", `String network_id); ("nodes", `List nodes) ] -> + Ok (List.fold nodes ~init:Core.String.Map.empty ~f:(f network_id)) | t -> raise @@ Invalid_output (Yojson.Safe.to_string t) end @@ -529,7 +530,6 @@ module Command_output = struct end module Arg_type = struct - (* TODO: what other arg types? *) let bool = "bool" let int = "int" @@ -550,124 +550,139 @@ module Arg_type = struct raise @@ Invalid_arg_type t end -let version config = - match Yojson.Safe.Util.member "version" config with - | `Int version -> - version - | t -> - raise @@ Config_file.Invalid_version t - -let action name config = - let open Yojson.Safe in - match Util.member "actions" config with - | `List actions -> - List.find_exn actions ~f:(fun action -> - Util.member "name" action |> equal @@ `String name ) - | t -> - raise @@ Config_file.Invalid_actions t - -let args_of_action action = - let open Yojson.Safe in - let a_list = - match Util.member "args" action with - | `Assoc a_list -> - a_list +module Config_file = struct + type t = { version : int; actions : action list } [@@deriving eq, yojson] + + and action = { name : string; args : Yojson.Safe.t; command : string } + [@@deriving eq, yojson] + + exception Invalid_version of Yojson.Safe.t + + exception Invalid_actions of Yojson.Safe.t + + exception Invalid_args of Yojson.Safe.t + + exception Invalid_arg_type_should_be_string of string * Yojson.Safe.t + + exception Invalid_command of Yojson.Safe.t + + exception Invalid_args_num + + exception Missing_arg of string * string + + exception Invalid_arg_type of string * Yojson.Safe.t * string + + let version config = + match Yojson.Safe.Util.member "version" config with + | `Int version -> + version | t -> - raise @@ Config_file.Invalid_args t - in - List.map a_list ~f:(function - | name, `String type_name -> - (name, type_name) - | name, t -> - raise @@ Config_file.Invalid_arg_type_should_be_string (name, t) ) - -let raw_cmd t = - match Yojson.Safe.Util.member "command" t with - | `String raw_cmd -> - raw_cmd - | cmd -> - raise @@ Config_file.Invalid_command cmd - -let validate_arg_type arg_typ arg_value = - String.equal arg_typ @@ Arg_type.arg_type arg_value - -exception Invalid_config_args_num - -exception Missing_config_arg of string * string - -exception Invalid_config_arg_type of string * Yojson.Safe.t * string - -let validate_args ~args ~action = - let arg_list = args_of_action action in - let () = - if not List.(length arg_list = length args) then - raise @@ Invalid_config_args_num - in - let rec aux = function + raise @@ Invalid_version t + + let action name config = + let open Yojson.Safe in + match Util.member "actions" config with + | `List actions -> + List.find_exn actions ~f:(fun action -> + Util.member "name" action |> equal @@ `String name ) + | t -> + raise @@ Invalid_actions t + + let args_of_action action = + let open Yojson.Safe in + let a_list = + match Util.member "args" action with + | `Assoc a_list -> + a_list + | t -> + raise @@ Invalid_args t + in + List.map a_list ~f:(function + | name, `String type_name -> + (name, type_name) + | name, t -> + raise @@ Invalid_arg_type_should_be_string (name, t) ) + + let raw_cmd t = + match Yojson.Safe.Util.member "command" t with + | `String raw_cmd -> + raw_cmd + | cmd -> + raise @@ Invalid_command cmd + + let validate_arg_type arg_typ arg_value = + String.equal arg_typ @@ Arg_type.arg_type arg_value + + let validate_args ~args ~action = + let arg_list = args_of_action action in + let () = + if not List.(length arg_list = length args) then raise @@ Invalid_args_num + in + let rec aux = function + | [] -> + true + | (arg_name, arg_type) :: rest -> + let arg_value = + try + List.find_map_exn args ~f:(fun (name, value) -> + Option.some_if (String.equal name arg_name) value ) + with Not_found_s _ -> raise @@ Missing_arg (arg_name, arg_type) + in + if validate_arg_type arg_type arg_value then aux rest + else raise @@ Invalid_arg_type (arg_name, arg_value, arg_type) + in + aux arg_list + + let rec interpolate_args ~args raw_cmd = + match args with | [] -> - true - | (arg_name, arg_type) :: rest -> - let arg_value = - try - List.find_map_exn args ~f:(fun (name, value) -> - Option.some_if (String.equal name arg_name) value ) - with Not_found_s _ -> - raise @@ Missing_config_arg (arg_name, arg_type) - in - if validate_arg_type arg_type arg_value then aux rest - else raise @@ Invalid_config_arg_type (arg_name, arg_value, arg_type) - in - aux arg_list - -let rec interpolate_args ~args raw_cmd = - match args with - | [] -> - raw_cmd - | (arg, value) :: args -> - let pattern = sprintf "{{%s}}" arg in - interpolate_args ~args - @@ String.substr_replace_all raw_cmd ~pattern - ~with_:(Yojson.Safe.to_string value |> Util.drop_outer_quotes) - -let prog_and_args ~args action = - let raw_cmd = raw_cmd action in - let cmd_list = interpolate_args ~args raw_cmd |> String.split ~on:' ' in - (List.hd_exn cmd_list, List.tl_exn cmd_list) - -let run_command ?(suppress_logs = false) ?(timeout_seconds = 1) ?(dir = ".") - ~config ~args cmd_name = - let config = Yojson.Safe.from_file config in - let version = version config in - let action = action cmd_name config in - let action_args = args_of_action action in - let () = assert (validate_args ~args ~action) in - let prog, arg_values = prog_and_args ~args action in - let cmd = String.concat ~sep:" " (prog :: arg_values) in - let%map output = - Util.run_cmd_or_error_timeout ~suppress_logs ~timeout_seconds dir prog - arg_values - in - match output with - | Ok output -> - if not suppress_logs then - [%log' spam (Logger.create ())] - "Successful command execution\nCommand: %s\nOutput: %s" cmd output ; - Ok output - | _ -> - if not suppress_logs then - [%log' error (Logger.create ())] "Failed to run command: %s" cmd ; - Error - (sprintf - "Failed to run command: %s\n\ - \ Config version: %d\n\ - \ Expected args types:\n\ - \ %s\n\ - \ Attempted args:\n\ - \ %s\n" - cmd version - ( String.concat ~sep:", " - @@ List.map action_args ~f:(fun (name, type_name) -> - name ^ ": " ^ type_name ) ) - ( String.concat ~sep:", " - @@ List.map args ~f:(fun (name, value) -> - name ^ ": " ^ Yojson.Safe.to_string value ) ) ) + raw_cmd + | (arg, value) :: args -> + let pattern = sprintf "{{%s}}" arg in + interpolate_args ~args + @@ String.substr_replace_all raw_cmd ~pattern + ~with_:(Yojson.Safe.to_string value |> Util.drop_outer_quotes) + + let prog_and_args ~args action = + let raw_cmd = raw_cmd action in + let cmd_list = interpolate_args ~args raw_cmd |> String.split ~on:' ' in + (List.hd_exn cmd_list, List.tl_exn cmd_list) + + let run_command ?(suppress_logs = false) ?(timeout_seconds = 1) ?(dir = ".") + ~config ~args cmd_name = + let config = Yojson.Safe.from_file config in + let version = version config in + let action = action cmd_name config in + let action_args = args_of_action action in + let () = assert (validate_args ~args ~action) in + let prog, arg_values = prog_and_args ~args action in + let cmd = String.concat ~sep:" " (prog :: arg_values) in + let%map output = + Util.run_cmd_or_error_timeout ~suppress_logs ~timeout_seconds dir prog + arg_values + in + match output with + | Ok output -> + if not suppress_logs then + [%log' spam (Logger.create ())] + "Successful command execution\nCommand: %s\nOutput: %s" cmd output ; + Ok output + | _ -> + if not suppress_logs then + [%log' error (Logger.create ())] "Failed to run command: %s" cmd ; + Error + (sprintf + "Failed to run command: %s\n\ + \ Config version: %d\n\ + \ Expected args types:\n\ + \ %s\n\ + \ Attempted args:\n\ + \ %s\n" + cmd version + ( String.concat ~sep:", " + @@ List.map action_args ~f:(fun (name, type_name) -> + name ^ ": " ^ type_name ) ) + ( String.concat ~sep:", " + @@ List.map args ~f:(fun (name, value) -> + name ^ ": " ^ Yojson.Safe.to_string value ) ) ) +end diff --git a/src/lib/integration_test_abstract_engine/config.json b/src/lib/integration_test_abstract_engine/config.json index afe6a914eb1..10ad130694a 100644 --- a/src/lib/integration_test_abstract_engine/config.json +++ b/src/lib/integration_test_abstract_engine/config.json @@ -2,9 +2,23 @@ "version": 1, "actions": [ { - "name": "start_network", + "name": "create_network", "args": {}, - "command": "minimina network start" + "command": "minimina network create" + }, + { + "name": "deploy_network", + "args": { + "network_id":"string" + }, + "command": "minimina network deploy --network-id {{network_id}}" + }, + { + "name": "destroy_network", + "args": { + "network_id":"string" + }, + "command": "minimina network destroy --network-id {{network_id}}" }, { "name": "get_network_status", @@ -14,12 +28,54 @@ "command": "minimina network status --network-id {{network_id}}" }, { - "name": "get_logs", + "name": "start_node", + "args": { + "network_id":"string", + "node_id":"string", + "fresh_state":"bool", + "git_commit":"string" + }, + "command": "minimina node start --network-id {{network_id}} --node-id {{node_id}} --fresh-state {{fresh_state}} --git-commit {{git_commit}}" + }, + { + "name": "stop_node", + "args": { + "network_id":"string", + "node_id":"string" + }, + "command": "minimina node stop --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "dump_archive_data", + "args": { + "network_id":"string", + "node_id":"string" + }, + "command": "minimina node dump-archive-data --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "dump_mina_logs", + "args": { + "network_id":"string", + "node_id":"string" + }, + "command": "minimina node dump-mina-logs --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "dump_precomputed_blocks", + "args": { + "network_id":"string", + "node_id":"string" + }, + "command": "minimina node dump-precomputed-blocks --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "run_replayer", "args": { - "network_id": "string", - "node_id": "string" + "network_id":"string", + "node_id":"string" }, - "command": "minimina node logs --node-id {{node_id}} --network-id {{network_id}}" + "command": "minimina node run-replayer --network-id {{network_id}} --node-id {{node_id}}" }, { "name": "echo", diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index c5f0ac7ac62..1def2d691f7 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -25,7 +25,7 @@ module Network_manager = struct let create ~logger (network_config : Network_config.t) = let open Malleable_error.Let_syntax in let%bind current_cluster = - (* TODO: replace with run_command *) + (* TODO: replace with Config_file.run_command *) Util.run_cmd_or_hard_error "/" "kubectl" [ "config"; "current-context" ] in [%log info] "Using cluster: %s" current_cluster ; @@ -52,7 +52,7 @@ module Network_manager = struct "Existing namespace of same name detected; removing to start \ clean" in - (* TODO: replace with run_command *) + (* TODO: replace with Config_file.run_command *) Util.run_cmd_or_hard_error "/" "kubectl" [ "delete"; "namespace"; network_config.config.network_id ] >>| Fn.const () @@ -104,13 +104,13 @@ module Network_manager = struct (* TODO: use output *) let deploy t = + let open Network_deployed in let logger = t.logger in if t.deployed then failwith "network already deployed" ; [%log info] "Deploying network" ; - let%bind output = - let open Network_deployed in + let%bind network_deployed = match%map - run_command + Config_file.run_command ~config:!Abstract_network.config_path ~args:[] "deploy_network" with @@ -120,39 +120,33 @@ module Network_manager = struct | Error err -> raise @@ Invalid_output err in - let _ = Map.is_empty output in t.deployed <- true ; - let seeds = Core.String.Map.empty in - let block_producers = Core.String.Map.empty in - let snark_coordinators = Core.String.Map.empty in - let snark_workers = Core.String.Map.empty in - let archive_nodes = Core.String.Map.empty in let network = - { Abstract_network.constants = t.constants + let open Abstract_network in + { constants = t.constants ; testnet_log_filter = t.testnet_log_filter ; genesis_keypairs = t.genesis_keypairs - ; seeds - ; block_producers - ; snark_coordinators - ; snark_workers - ; archive_nodes + ; archive_nodes = Node.Collections.archive_nodes network_deployed + ; block_producers = Node.Collections.block_producers network_deployed + ; seeds = Node.Collections.seeds network_deployed + ; snark_coordinators = + Node.Collections.snark_coordinators network_deployed + ; snark_workers = Node.Collections.snark_workers network_deployed ; network_id = t.network_id } in - let nodes_to_string = - Fn.compose (String.concat ~sep:", ") - (List.map ~f:Abstract_network.Node.id) + let nodes_to_string nodes = + Core.String.Map.data nodes + |> Fn.compose (String.concat ~sep:", ") + (List.map ~f:Abstract_network.Node.id) in [%log info] "Network deployed" ; [%log info] "network id: %s" t.network_id ; [%log info] "snark coordinators: %s" - (nodes_to_string (Core.String.Map.data network.snark_coordinators)) ; - [%log info] "snark workers: %s" - (nodes_to_string (Core.String.Map.data network.snark_workers)) ; - [%log info] "block producers: %s" - (nodes_to_string (Core.String.Map.data network.block_producers)) ; - [%log info] "archive nodes: %s" - (nodes_to_string (Core.String.Map.data network.archive_nodes)) ; + (nodes_to_string network.snark_coordinators) ; + [%log info] "snark workers: %s" (nodes_to_string network.snark_workers) ; + [%log info] "block producers: %s" (nodes_to_string network.block_producers) ; + [%log info] "archive nodes: %s" (nodes_to_string network.archive_nodes) ; Malleable_error.return network let destroy t = diff --git a/src/lib/integration_test_abstract_engine/unit_tests.ml b/src/lib/integration_test_abstract_engine/unit_tests.ml index 26585d5ed7f..361c4ccfb99 100644 --- a/src/lib/integration_test_abstract_engine/unit_tests.ml +++ b/src/lib/integration_test_abstract_engine/unit_tests.ml @@ -63,7 +63,9 @@ end open Test_values -module Helper_tests = struct +module Config_tests = struct + open Config_file + let%test_unit "version" = let res = version config in let expect = 1 in @@ -119,6 +121,8 @@ module Helper_tests = struct end module Run_command_tests = struct + open Config_file + module Arg_failures = struct let ( = ) = String.equal @@ -126,7 +130,7 @@ module Run_command_tests = struct let arg, value = ("msg", `Int 42) in try ignore @@ run_command ~config:config_file ~args:[ (arg, value) ] "echo" - with Invalid_config_arg_type (arg_name, arg_value, arg_type) -> + with Invalid_arg_type (arg_name, arg_value, arg_type) -> assert ( arg_name = "msg" && arg_type = Arg_type.string && Yojson.Safe.equal arg_value value ) @@ -137,7 +141,7 @@ module Run_command_tests = struct @@ run_command ~config:config_file ~args:[ ("msg", `String "hello"); ("num", `Int 42) ] "echo" - with Invalid_config_args_num -> () + with Invalid_args_num -> () let%test_unit "run command missing arg failure" = try @@ -145,7 +149,7 @@ module Run_command_tests = struct @@ run_command ~config:config_file ~args:[ ("msg0", `String "hello") ] "echo" - with Missing_config_arg (arg_name, arg_type) -> + with Missing_arg (arg_name, arg_type) -> assert (arg_name = "msg" && arg_type = Arg_type.string) end @@ -249,31 +253,39 @@ module Parse_output_tests = struct let open Network_deployed in let result = {| - { "node0": { - "node_type":"Archive_node", - "private_key":null, - "graphql_uri":"gql_archive" - }, - "node1": { - "node_type":"Block_producer_node", - "private_key":null, - "graphql_uri":"gql_bp" - }, - "node2": { - "node_type":"Seed_node", - "private_key":"EKEQpDAjj7dP3j7fQy4qBU7Kxns85wwq5xMn4zxdyQm83pEWzQ62", - "graphql_uri":"gql_seed" - }, - "node3": { - "node_type":"Snark_worker", - "private_key":null, - "graphql_uri":"gql_snark" - }, - "node4": { - "node_type":"Snark_coordinator", - "private_key":null, - "graphql_uri":null - } + { "network_id":"network0", + "nodes": [ + { "node0": { + "node_type":"Archive_node", + "private_key":null, + "graphql_uri":"gql_archive" + } + }, + { "node1": { + "node_type":"Block_producer_node", + "private_key":null, + "graphql_uri":"gql_bp" + } + }, + { "node2": { + "node_type":"Seed_node", + "private_key":"EKEQpDAjj7dP3j7fQy4qBU7Kxns85wwq5xMn4zxdyQm83pEWzQ62", + "graphql_uri":"gql_seed" + } + }, + { "node3": { + "node_type":"Snark_worker", + "private_key":null, + "graphql_uri":"gql_snark" + } + }, + { "node4": { + "node_type":"Snark_coordinator", + "private_key":null, + "graphql_uri":null + } + } + ] } |} |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith @@ -291,37 +303,47 @@ module Parse_output_tests = struct Network_keypair.create_network_keypair ~keypair_name:"node2_key" ~keypair in let archive = - { node_type = Archive_node + { node_id = "node0" + ; network_id = "network0" + ; node_type = Archive_node ; network_keypair = None ; graphql_uri = Some "gql_archive" } in let bp = - { node_type = Block_producer_node + { node_id = "node1" + ; node_type = Block_producer_node + ; network_id = "network0" ; network_keypair = None ; graphql_uri = Some "gql_bp" } in let seed = - { node_type = Seed_node + { node_id = "node2" + ; node_type = Seed_node + ; network_id = "network0" ; network_keypair = Some seed_keypair ; graphql_uri = Some "gql_seed" } in let worker = - { node_type = Snark_worker + { node_id = "node3" + ; node_type = Snark_worker + ; network_id = "network0" ; network_keypair = None ; graphql_uri = Some "gql_snark" } in let coordinator = - { node_type = Snark_coordinator + { node_id = "node4" + ; node_type = Snark_coordinator + ; network_id = "network0" ; network_keypair = None ; graphql_uri = None } in let expect = - of_alist_exn + Core.String.Map.of_alist_exn [ ("node0", archive) ; ("node1", bp) ; ("node2", seed) @@ -329,12 +351,7 @@ module Parse_output_tests = struct ; ("node4", coordinator) ] in - assert ( - (* print_endline "=== Result ===" ; - Map.iter result ~f:(fun ni -> node_info_to_yojson ni |> Yojson.Safe.to_string |> print_endline) ; - print_endline "\n\n=== Expect ===" ; - Map.iter expect ~f:(fun ni -> node_info_to_yojson ni |> Yojson.Safe.to_string |> print_endline) ; *) - equal result expect ) + assert (equal result expect) let%test_unit "parse node started" = let open Node_started in From f264b39bdad8519f4ce23855e10c23746ed6d647 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Thu, 3 Aug 2023 02:50:04 +0000 Subject: [PATCH 25/90] (test exec) Log use of config with non-abstract engine --- src/app/test_executive/test_executive.ml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index 4fedf9b7a64..8f36aa329d8 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -49,13 +49,18 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : let (Test_inputs_with_cli_inputs ((module Inputs), _)) = inputs.test_inputs in - if - String.equal Inputs.Engine.name "abstract" - && Option.is_none inputs.config_path - then ( - [%log fatal] "Must provide a config file when using the abstract engine" ; - exit 1 ) - else Deferred.unit + match Inputs.Engine.name with + | "abstract" -> + if Option.is_none inputs.config_path then ( + [%log fatal] + "Must provide a config file when using the abstract engine" ; + exit 1 ) + else Deferred.unit + | _ -> + [%log debug] + "Config file is only used for the abstract engine. It will be \ + ignored." ; + Deferred.unit let engines : engine list = [ ("abstract", (module Integration_test_abstract_engine : Intf.Engine.S)) From f31222bcb59d6de69a825c0cde8a6c6ac7db2f98 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Thu, 3 Aug 2023 02:51:05 +0000 Subject: [PATCH 26/90] (config) Add network status response and unit tests --- .../abstract_network.ml | 8 ----- .../ci_interaction.ml | 29 ++++++++++++++-- .../unit_tests.ml | 34 +++++++++++++++++-- 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index 4b8593e9694..1f7aba44d37 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -24,14 +24,6 @@ module Node = struct let network_keypair { network_keypair; _ } = network_keypair - (* TODO: remove *) - let[@warning "-27"] get_logs_in_container ?container_id { node_id; _ } = - (* let container_id = Option.value container_id ~default:"" in - let%bind cwd = Unix.getcwd () in - Integration_test_lib.Util.run_cmd_or_hard_error ~exit_code:13 cwd "kubectl" - (base_kube_args config @ [ "logs"; "-c"; container_id; node_id ]) *) - failwith "get_logs_in_container" - let start ?(git_commit = Mina_version.commit_id_short) ~fresh_state t : unit Malleable_error.t = try diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 7f7beb0d8c9..5033564c37c 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -515,11 +515,36 @@ module Replayer_run = struct exception Invalid_response of string end +module Network_status = struct + type t = Deploy_error of string | Status of Node_id.t Core.String.Map.t + [@@deriving eq] + + exception Invalid_status of string * string + + let of_yojson (t : Yojson.Safe.t) = + match t with + | `Assoc [ ("deploy_error", `String error) ] -> + Ok (Deploy_error error) + | `Assoc statuses -> + let status_map = + let open Core.String.Map in + List.fold statuses ~init:empty ~f:(fun acc -> function + | node_id, `String status -> + set acc ~key:node_id ~data:status + | node_id, error -> + raise @@ Invalid_status (node_id, Yojson.Safe.to_string error) ) + in + Ok (Status status_map) + | _ -> + Error (Yojson.Safe.to_string t) +end + module Command_output = struct type t = | Network_created of Network_created.t | Network_deployed of Network_deployed.t | Network_destroyed + | Network_status of Network_status.t | Node_started of Node_started.t | Node_stopped of Node_stopped.t | Archive_data_dump of Archive_data_dump.t @@ -665,7 +690,7 @@ module Config_file = struct | Ok output -> if not suppress_logs then [%log' spam (Logger.create ())] - "Successful command execution\nCommand: %s\nOutput: %s" cmd output ; + "Successful command execution\n Command: %s\n Output: %s" cmd output ; Ok output | _ -> if not suppress_logs then @@ -684,5 +709,5 @@ module Config_file = struct name ^ ": " ^ type_name ) ) ( String.concat ~sep:", " @@ List.map args ~f:(fun (name, value) -> - name ^ ": " ^ Yojson.Safe.to_string value ) ) ) + name ^ " = " ^ Yojson.Safe.to_string value ) ) ) end diff --git a/src/lib/integration_test_abstract_engine/unit_tests.ml b/src/lib/integration_test_abstract_engine/unit_tests.ml index 361c4ccfb99..101a82303fe 100644 --- a/src/lib/integration_test_abstract_engine/unit_tests.ml +++ b/src/lib/integration_test_abstract_engine/unit_tests.ml @@ -386,7 +386,7 @@ module Parse_output_tests = struct in assert (equal result { node_id = "node0"; logs }) - let%test_unit "parse precomputed block dump response" = + let%test_unit "parse precomputed block dump" = let open Precomputed_block_dump in let result = {|{"node_id":"node0","blocks":"blocks"}|} |> Yojson.Safe.from_string @@ -394,7 +394,7 @@ module Parse_output_tests = struct in assert (equal result { node_id = "node0"; blocks = "blocks" }) - let%test_unit "Replayer run response" = + let%test_unit "parse replayer run" = let open Replayer_run in let logs = "{\"log0\":\"msg0\"}\n{\"log1\":\"msg1\"}" in let result = @@ -402,6 +402,36 @@ module Parse_output_tests = struct |> of_yojson |> Result.ok_or_failwith in assert (equal result { node_id = "node0"; logs }) + + let%test_unit "parse network status deploy error" = + let open Network_status in + let err_msg = "this is an error msg" in + let deploy_error = sprintf {|{"deploy_error":"%s"}|} err_msg in + let res = + Yojson.Safe.from_string deploy_error |> of_yojson |> Result.ok_or_failwith + in + assert (equal res @@ Deploy_error err_msg) + + let%test_unit "parse network status" = + let open Network_status in + let node0 = "node0" in + let node1 = "node1" in + let status0 = sprintf "%s's status" node0 in + let status1 = sprintf "%s's status" node1 in + let status = + sprintf {|{"%s":"%s", "%s":"%s"}|} node0 status0 node1 status1 + in + let res = + Yojson.Safe.from_string status |> of_yojson |> Result.ok_or_failwith + in + let expect_status_map = + let open Core.String.Map in + List.fold + [ (node0, status0); (node1, status1) ] + ~init:empty + ~f:(fun acc (key, data) -> set acc ~key ~data) + in + assert (equal res @@ Status expect_status_map) end let () = ignore @@ Async.Scheduler.go () From 69d9190463707aaa75ebf22ad0951ce69b847c26 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sun, 13 Aug 2023 05:01:42 +0000 Subject: [PATCH 27/90] (config) Add return spec --- integration_tests/return_spec.json | 51 ++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 integration_tests/return_spec.json diff --git a/integration_tests/return_spec.json b/integration_tests/return_spec.json new file mode 100644 index 00000000000..9c9df4b4b20 --- /dev/null +++ b/integration_tests/return_spec.json @@ -0,0 +1,51 @@ +{ + "version": 1, + "return_spec": { + "create_network": { + "network_id": "string", + "nodes_map": [ + { + "graphql_uri" : "string", + "network_id" : "string", + "network_keypair" : "string", + "node_id" : "string", + "node_type" : "string" + } + ] + }, + "start_network": { + "network_id": "string" + }, + "delete_network": { + "network_id": "string" + }, + "start_node": { + "network_id": "string", + "node_id": "string" + }, + "stop_node": { + "network_id": "string", + "node_id": "string" + }, + "dump_archive_data": { + "data": "string", + "network_id": "string", + "node_id": "string" + }, + "dump_mina_logs": { + "logs": "string", + "network_id": "string", + "node_id": "string" + }, + "dump_precomputed_block": { + "blocks": "string", + "network_id": "string", + "node_id": "string" + }, + "run_replayer": { + "logs": "string", + "network_id": "string", + "node_id": "string" + } + } +} From 378fe7cd760e6599f8d8284e68cdaabeb118884d Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sun, 13 Aug 2023 05:03:35 +0000 Subject: [PATCH 28/90] (util) Add option to extend child process env --- src/lib/integration_test_lib/util.ml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/integration_test_lib/util.ml b/src/lib/integration_test_lib/util.ml index 341ee4eb8e0..1a346cb1132 100644 --- a/src/lib/integration_test_lib/util.ml +++ b/src/lib/integration_test_lib/util.ml @@ -54,14 +54,14 @@ let check_cmd_output ~prog ~args output = Or_error.errorf "command exited prematurely due to signal %d" (Signal.to_system_int signal) -let run_cmd_or_error_timeout ?(suppress_logs = false) ~timeout_seconds dir prog - args = +let run_cmd_or_error_timeout ?(suppress_logs = false) ?(env = `Extend []) + ~timeout_seconds dir prog args = if not suppress_logs then [%log' spam (Logger.create ())] "Running command (from %s): $command" dir ~metadata:[ ("command", `String (String.concat (prog :: args) ~sep:" ")) ] ; let open Deferred.Let_syntax in - let%bind process = Process.create_exn ~working_dir:dir ~prog ~args () in + let%bind process = Process.create_exn ~working_dir:dir ~env ~prog ~args () in let%bind res = match%map Timeout.await () From c4e1f8f596b43f0a2d5dd04667b4953ecf181711 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sun, 13 Aug 2023 05:04:10 +0000 Subject: [PATCH 29/90] (unit test) Update with minimina commands --- .../unit_tests.ml | 94 +++++++++++++++---- 1 file changed, 77 insertions(+), 17 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/unit_tests.ml b/src/lib/integration_test_abstract_engine/unit_tests.ml index 101a82303fe..e308c7b581b 100644 --- a/src/lib/integration_test_abstract_engine/unit_tests.ml +++ b/src/lib/integration_test_abstract_engine/unit_tests.ml @@ -16,21 +16,42 @@ module Test_values = struct "minimina network deploy --network-id {{network_id}}" let start_node_raw_cmd = - "minimina start-node --node-id {{node_id}} --fresh-state {{fresh_state}} \ - --git-commit {{git_commit}}" + "minimina node start --network-id {{network_id}} --node-id {{node_id}} \ + --fresh-state {{fresh_state}} --git-commit {{git_commit}}" let deploy_network_action = sprintf {| + { + "name": "deploy_network", + "args": { + "network_id": "string" + }, + "returns": [ + { + "graphql_uri" : "string", + "network_id" : "string", + "network_keypair" : "string", + "node_id" : "string", + "node_type" : "string" + } + ], + "command": "%s" + } + |} + deploy_network_raw_cmd + |> String.strip + + let json_deploy_network_response = + {| { - "name": "deploy_network", - "args": { - "network_id": "string" - }, - "command": "%s" + "graphql_uri" : "", + "network_id" : "network0", + "network_keypair" : "EKEQpDAjj7dP3j7fQy4qBU7Kxns85wwq5xMn4zxdyQm83pEWzQ62", + "node_id" : "node0", + "node_type" : "Archive_node" } |} - deploy_network_raw_cmd |> String.strip let start_node_action = @@ -39,6 +60,7 @@ module Test_values = struct { "name": "start_node", "args": { + "network_id": "string", "node_id": "string", "fresh_state": "bool", "git_commit": "string" @@ -88,13 +110,16 @@ module Config_tests = struct let%test_unit "validate args" = let action = action "start_node" config in let args = - [ ("node_id", `String "node0") + [ ("network_id", `String "network0") + ; ("node_id", `String "node0") ; ("fresh_state", `Bool true) ; ("git_commit", `String "0123456abcdef") ] in assert (validate_args ~args ~action) + let%test_unit "TODO: validate_cmd_args" = assert false + let%test_unit "interpolate string args" = let res = interpolate_args @@ -104,17 +129,49 @@ module Config_tests = struct let expect = {|minimina network deploy --network-id network0|} in assert (res = expect) + (* let%test_unit "command returns" = + let returns = + returns_of_action @@ Yojson.Safe.from_string deploy_network_action + in + let[@warning "-8"] (Record record) = returns in + let expect = + [ ("graphql_uri", `Null) + ; ("network_id", `String "network0") + ; ( "network_keypair" + , `String "EKEQpDAjj7dP3j7fQy4qBU7Kxns85wwq5xMn4zxdyQm83pEWzQ62" ) + ; ("node_id", `String "node0") + ; ("node_type", `String "Archive_node") + ] + in + let ( = ) = + List.equal (fun (a, b) (c, d) -> + String.equal a c && Yojson.Safe.equal b d ) + in + assert (record = expect) *) + let%test_unit "prog and args" = let action = Yojson.Safe.from_string start_node_action in let args = - [ ("node_id", `String "node0") + [ ("network_id", `String "network0") + ; ("node_id", `String "node0") ; ("fresh_state", `Bool true) - ; ("commit_sha", `String "0123456abcdef") + ; ("git_commit", `String "0123456abcdef") ] in let prog, res_args = prog_and_args ~args action in let expect = - String.split ~on:' ' @@ interpolate_args ~args start_node_raw_cmd + [ "minimina" + ; "node" + ; "start" + ; "--network-id" + ; "network0" + ; "--node-id" + ; "node0" + ; "--fresh-state" + ; "true" + ; "--git-commit" + ; "0123456abcdef" + ] in assert (prog = List.hd_exn expect) ; assert (List.equal ( = ) res_args @@ List.tl_exn expect) @@ -141,7 +198,12 @@ module Run_command_tests = struct @@ run_command ~config:config_file ~args:[ ("msg", `String "hello"); ("num", `Int 42) ] "echo" - with Invalid_args_num -> () + with Invalid_args_num (action_name, input_args, action_args) -> + let open String in + let input_args = concat ~sep:", " input_args in + let action_args = concat ~sep:", " action_args in + printf "Action: %s\n Input args: %s\n Expected args: %s\n" action_name + input_args action_args let%test_unit "run command missing arg failure" = try @@ -157,19 +219,17 @@ module Run_command_tests = struct open Async.Deferred.Let_syntax let echo_command = - let arg, value = ("msg", `String "hello") in let%map output = run_command ~suppress_logs:true ~config:config_file - ~args:[ (arg, value) ] + ~args:[ ("msg", `String "hello") ] "echo" in if Result.is_error output then assert false let cat_command = - let arg, value = ("path", `String config_file) in match%map run_command ~suppress_logs:true ~config:config_file - ~args:[ (arg, value) ] + ~args:[ ("path", `String config_file) ] "cat" with | Ok output -> From de040ef0a39f662415ce6a13ae0dd004976c442b Mon Sep 17 00:00:00 2001 From: Quantifier Date: Mon, 14 Aug 2023 20:23:29 +0000 Subject: [PATCH 30/90] (mock) Add mock config and test for abstract engine --- integration_tests/mock_config.json | 78 ++++++++++++++++++++++++++++++ src/app/test_executive/mock.ml | 54 +++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 integration_tests/mock_config.json create mode 100644 src/app/test_executive/mock.ml diff --git a/integration_tests/mock_config.json b/integration_tests/mock_config.json new file mode 100644 index 00000000000..501508de9fc --- /dev/null +++ b/integration_tests/mock_config.json @@ -0,0 +1,78 @@ +{ + "version": 1, + "actions": [ + { + "name": "create_network", + "args": { + "network_id": "string", + "test_config": "string", + "topology": "string" + }, + "command": "MOCK_NETWORK network create --network-id {{network_id}} --test-config {{test_config}}" + }, + { + "name": "start_network", + "args": { + "network_id": "string" + }, + "command": "MOCK_NETWORK network start --network-id {{network_id}}" + }, + { + "name": "delete_network", + "args": { + "network_id": "string" + }, + "command": "MOCK_NETWORK network delete --network-id {{network_id}}" + }, + { + "name": "start_node", + "args": { + "network_id": "string", + "node_id": "string", + "fresh_state": "bool", + "git_commit": "string" + }, + "command": "MOCK_NETWORK node start --fresh-state {{fresh_state}} --git-commit {{git_commit}} --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "stop_node", + "args": { + "network_id": "string", + "node_id": "string" + }, + "command": "MOCK_NETWORK node stop --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "dump_archive_data", + "args": { + "network_id": "string", + "node_id": "string" + }, + "command": "MOCK_NETWORK node dump-archive-data --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "dump_mina_logs", + "args": { + "network_id": "string", + "node_id": "string" + }, + "command": "MOCK_NETWORK node dump-mina-logs --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "dump_precomputed_blocks", + "args": { + "network_id": "string", + "node_id": "string" + }, + "command": "MOCK_NETWORK node dump-precomputed-blocks --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "run_replayer", + "args": { + "network_id": "string", + "node_id": "string" + }, + "command": "MOCK_NETWORK node run-replayer --network-id {{network_id}} --node-id {{node_id}}" + } + ] +} diff --git a/src/app/test_executive/mock.ml b/src/app/test_executive/mock.ml new file mode 100644 index 00000000000..651d7205c8a --- /dev/null +++ b/src/app/test_executive/mock.ml @@ -0,0 +1,54 @@ +open Core +open Integration_test_lib + +module Make (Inputs : Intf.Test.Inputs_intf) = struct + open Test_common.Make (Inputs) + + let num_extra_keys = 1000 + + let test_name = "mock" + + let config = + let open Test_config in + { default with + genesis_ledger = + [ { Test_Account.account_name = "receiver-key" + ; balance = "9999999" + ; timing = Untimed + } + ; { account_name = "empty-bp-key"; balance = "0"; timing = Untimed } + ; { account_name = "snark-node-key"; balance = "0"; timing = Untimed } + ] + @ List.init num_extra_keys ~f:(fun i -> + let i_str = Int.to_string i in + { Test_Account.account_name = + String.concat [ "sender-account"; i_str ] + ; balance = "10000" + ; timing = Untimed + } ) + ; block_producers = + [ { node_name = "receiver"; account_name = "receiver-key" } + ; { node_name = "empty_node-1"; account_name = "empty-bp-key" } + ; { node_name = "empty_node-2"; account_name = "empty-bp-key" } + ; { node_name = "empty_node-3"; account_name = "empty-bp-key" } + ; { node_name = "empty_node-4"; account_name = "empty-bp-key" } + ; { node_name = "observer"; account_name = "empty-bp-key" } + ] + ; snark_coordinator = + Some + { node_name = "snark-node" + ; account_name = "snark-node-key" + ; worker_nodes = 4 + } + ; txpool_max_size = 10_000_000 + ; snark_worker_fee = "0.0001" + ; proof_config = + { proof_config_default with + work_delay = Some 1 + ; transaction_capacity = + Some Runtime_config.Proof_keys.Transaction_capacity.small + } + } + + let run _network _t = Malleable_error.return () +end From e02c65e6e2956767e6d0dffc55b205726c65a710 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Mon, 14 Aug 2023 20:42:15 +0000 Subject: [PATCH 31/90] (test exec) Add alias command line arg and substitute in child process --- src/app/test_executive/test_executive.ml | 46 +++++++++++-------- .../abstract_network.ml | 5 +- .../kubernetes_network.ml | 2 + src/lib/integration_test_lib/intf.ml | 4 ++ src/lib/integration_test_lib/test_config.ml | 2 +- src/lib/integration_test_lib/util.ml | 6 +-- 6 files changed, 39 insertions(+), 26 deletions(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index 8f36aa329d8..57b431395fa 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -68,23 +68,20 @@ let engines : engine list = ] let tests : test list = - [ ( "peers-reliability" - , (module Peers_reliability_test.Make : Intf.Test.Functor_intf) ) - ; ( "chain-reliability" - , (module Chain_reliability_test.Make : Intf.Test.Functor_intf) ) - ; ("payments", (module Payments_test.Make : Intf.Test.Functor_intf)) - ; ("gossip-consis", (module Gossip_consistency.Make : Intf.Test.Functor_intf)) - ; ("medium-bootstrap", (module Medium_bootstrap.Make : Intf.Test.Functor_intf)) - ; ("zkapps", (module Zkapps.Make : Intf.Test.Functor_intf)) - ; ("zkapps-timing", (module Zkapps_timing.Make : Intf.Test.Functor_intf)) - ; ("zkapps-nonce", (module Zkapps_nonce_test.Make : Intf.Test.Functor_intf)) - ; ( "verification-key" - , (module Verification_key_update.Make : Intf.Test.Functor_intf) ) - ; ( "block-prod-prio" - , (module Block_production_priority.Make : Intf.Test.Functor_intf) ) - ; ("snarkyjs", (module Snarkyjs.Make : Intf.Test.Functor_intf)) - ; ("block-reward", (module Block_reward_test.Make : Intf.Test.Functor_intf)) - ; ("hard-fork", (module Hard_fork.Make : Intf.Test.Functor_intf)) + [ (module Block_production_priority.Make : Intf.Test.Functor_intf) + ; (module Block_reward_test.Make : Intf.Test.Functor_intf) + ; (module Chain_reliability_test.Make : Intf.Test.Functor_intf) + ; (module Delegation_test.Make : Intf.Test.Functor_intf) + ; (module Gossip_consistency.Make : Intf.Test.Functor_intf) + ; (module Medium_bootstrap.Make : Intf.Test.Functor_intf) + ; (module Mock.Make : Intf.Test.Functor_intf) + ; (module Payments_test.Make : Intf.Test.Functor_intf) + ; (module Peers_reliability_test.Make : Intf.Test.Functor_intf) + ; (module Snarkyjs.Make : Intf.Test.Functor_intf) + ; (module Verification_key_update.Make : Intf.Test.Functor_intf) + ; (module Zkapps.Make : Intf.Test.Functor_intf) + ; (module Zkapps_nonce_test.Make : Intf.Test.Functor_intf) + ; (module Zkapps_timing.Make : Intf.Test.Functor_intf) ] let report_test_errors ~log_error_set ~internal_error_set = @@ -434,6 +431,14 @@ let config_path_arg = & opt (some non_dir_file) None & info [ "config-path"; "config" ] ~env ~docv:"MINA_CI_CONFIG_PATH" ~doc) +let alias_arg = + let doc = "Alias to use for the mock network binary." in + let env = Arg.env_var "MOCK_NETWORK" ~doc in + Arg.( + value + & opt (some @@ pair ~sep:',' string string) None + & info [ "mock-network"; "mock"; "alias" ] ~env ~docv:"MOCK_NETWORK" ~doc) + let mina_image_arg = let doc = "Identifier of the Mina docker image to test." in let env = Arg.env_var "MINA_IMAGE" ~doc in @@ -475,10 +480,10 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = let info = info engine_name in let module Inputs = Make_test_inputs (Engine) () in let set_config path = - (* TODO: deal with path relativity? *) Option.iter path ~f:(fun p -> Engine.Network.config_path := p) ; path in + let set_alias alias = Engine.Network.alias := alias in let test_inputs_with_cli_inputs_arg = let wrap_cli_inputs cli_inputs = Test_inputs_with_cli_inputs ((module Inputs), cli_inputs) @@ -487,14 +492,15 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = in let inputs_term = let cons_inputs test_inputs test archive_image debug mina_image config_path - = + _ = { test_inputs; test; mina_image; archive_image; debug; config_path } in Term.( const cons_inputs $ test_inputs_with_cli_inputs_arg $ test_arg (module Inputs) $ archive_image_arg $ debug_arg $ mina_image_arg - $ (const set_config $ config_path_arg)) + $ (const set_config $ config_path_arg) + $ (const set_alias $ alias_arg)) in (Term.(const start $ inputs_term), info) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index 1f7aba44d37..2ca134bfebe 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -7,8 +7,9 @@ open Ci_interaction (* exclude from bisect_ppx to avoid type error on GraphQL modules *) [@@@coverage exclude_file] -(* [config_path] is instantiated when command line args are parsed *) -let config_path = ref "" +let config_path = config_path + +let alias = alias module Node = struct type t = diff --git a/src/lib/integration_test_cloud_engine/kubernetes_network.ml b/src/lib/integration_test_cloud_engine/kubernetes_network.ml index 1d5feb2c805..dbbbddb413c 100644 --- a/src/lib/integration_test_cloud_engine/kubernetes_network.ml +++ b/src/lib/integration_test_cloud_engine/kubernetes_network.ml @@ -5,6 +5,8 @@ open Integration_test_lib (* exclude from bisect_ppx to avoid type error on GraphQL modules *) [@@@coverage exclude_file] +let alias : (string * string) option ref = ref None + let config_path = ref "N/A" let id _ = "cloud" diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index 236d76d1a6b..0959dbdf2cf 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -105,6 +105,10 @@ module Engine = struct val id : t -> string + (*** [alias] is instantiated when command line args are parsed *) + val alias : (string * string) option ref + + (*** [config_path] is instantiated when command line args are parsed *) val config_path : string ref end diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 49e2281b86d..30d2328f58a 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -94,7 +94,7 @@ type t = ; snark_coordinator : Snark_coordinator_node.t option ; snark_worker_fee : string ; num_archive_nodes : int - ; log_precomputed_blocks : bool (* blockchain constants *) + ; log_precomputed_blocks : bool ; proof_config : Runtime_config.Proof_keys.t ; k : int ; delta : int diff --git a/src/lib/integration_test_lib/util.ml b/src/lib/integration_test_lib/util.ml index 1a346cb1132..341ee4eb8e0 100644 --- a/src/lib/integration_test_lib/util.ml +++ b/src/lib/integration_test_lib/util.ml @@ -54,14 +54,14 @@ let check_cmd_output ~prog ~args output = Or_error.errorf "command exited prematurely due to signal %d" (Signal.to_system_int signal) -let run_cmd_or_error_timeout ?(suppress_logs = false) ?(env = `Extend []) - ~timeout_seconds dir prog args = +let run_cmd_or_error_timeout ?(suppress_logs = false) ~timeout_seconds dir prog + args = if not suppress_logs then [%log' spam (Logger.create ())] "Running command (from %s): $command" dir ~metadata:[ ("command", `String (String.concat (prog :: args) ~sep:" ")) ] ; let open Deferred.Let_syntax in - let%bind process = Process.create_exn ~working_dir:dir ~env ~prog ~args () in + let%bind process = Process.create_exn ~working_dir:dir ~prog ~args () in let%bind res = match%map Timeout.await () From 07b8449f24f18d3311d03981383363e5b11fc8d6 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Mon, 14 Aug 2023 20:59:51 +0000 Subject: [PATCH 32/90] (unit test) Clean up config unit tests --- .../unit_tests.ml | 56 +++++++++++++------ 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/unit_tests.ml b/src/lib/integration_test_abstract_engine/unit_tests.ml index e308c7b581b..0f27d783c5a 100644 --- a/src/lib/integration_test_abstract_engine/unit_tests.ml +++ b/src/lib/integration_test_abstract_engine/unit_tests.ml @@ -415,53 +415,77 @@ module Parse_output_tests = struct let%test_unit "parse node started" = let open Node_started in + let network_id = "network0" in + let node_id = "node0" in let result = - {|{"node_id":"node0"}|} |> Yojson.Safe.from_string |> of_yojson - |> Result.ok_or_failwith + sprintf {|{"network_id":"%s", "node_id":"%s"}|} network_id node_id + |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith in - assert (equal result { node_id = "node0" }) + assert (equal result { network_id; node_id }) let%test_unit "parse node stopped" = let open Node_stopped in + let network_id = "network0" in + let node_id = "node0" in let result = - {|{"node_id":"node0"}|} |> Yojson.Safe.from_string |> of_yojson - |> Result.ok_or_failwith + sprintf {|{"network_id":"%s", "node_id":"%s"}|} network_id node_id + |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith in - assert (equal result { node_id = "node0" }) + assert (equal result { network_id; node_id }) let%test_unit "parse archive data dump" = let open Archive_data_dump in + let data = "datum0\ndatum1\ndatum2" in + let network_id = "network0" in + let node_id = "node0" in let result = - {|{"node_id":"node0","data":"data0"}|} |> Yojson.Safe.from_string - |> of_yojson |> Result.ok_or_failwith + sprintf {|{"network_id":"%s", "node_id":"%s", "data":"%s"}|} network_id + node_id data + |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith in - assert (equal result { node_id = "node0"; data = "data0" }) + assert (equal result { network_id; node_id; data }) let%test_unit "parse mina logs dump" = let open Mina_logs_dump in let logs = "{\"log0\":\"msg0\"}\n{\"log1\":\"msg1\"}" in + let network_id = "network0" in + let node_id = "node0" in let result = - `Assoc [ ("node_id", `String "node0"); ("logs", `String logs) ] + `Assoc + [ ("network_id", `String network_id) + ; ("node_id", `String node_id) + ; ("logs", `String logs) + ] |> of_yojson |> Result.ok_or_failwith in - assert (equal result { node_id = "node0"; logs }) + assert (equal result { network_id; node_id; logs }) let%test_unit "parse precomputed block dump" = let open Precomputed_block_dump in + let node_id = "node0" in + let network_id = "network0" in + let blocks = "block0\nblock1\nblocks2" in let result = - {|{"node_id":"node0","blocks":"blocks"}|} |> Yojson.Safe.from_string - |> of_yojson |> Result.ok_or_failwith + sprintf {|{"blocks":"%s", "network_id":"%s", "node_id":"%s"}|} blocks + network_id node_id + |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith in - assert (equal result { node_id = "node0"; blocks = "blocks" }) + assert (equal result { network_id; node_id; blocks }) let%test_unit "parse replayer run" = let open Replayer_run in + let node_id = "node0" in + let network_id = "network0" in let logs = "{\"log0\":\"msg0\"}\n{\"log1\":\"msg1\"}" in let result = - `Assoc [ ("node_id", `String "node0"); ("logs", `String logs) ] + `Assoc + [ ("network_id", `String network_id) + ; ("node_id", `String node_id) + ; ("logs", `String logs) + ] |> of_yojson |> Result.ok_or_failwith in - assert (equal result { node_id = "node0"; logs }) + assert (equal result { network_id; node_id; logs }) let%test_unit "parse network status deploy error" = let open Network_status in From 89aa58f064fd84916b5b68a62321d9329e89ea8e Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 16 Aug 2023 04:19:45 +0000 Subject: [PATCH 33/90] (config) Update configs --- integration_tests/mock_config.json | 5 ++--- .../integration_test_abstract_engine/config.json | 14 +++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/integration_tests/mock_config.json b/integration_tests/mock_config.json index 501508de9fc..3a3f7ea09cb 100644 --- a/integration_tests/mock_config.json +++ b/integration_tests/mock_config.json @@ -5,8 +5,7 @@ "name": "create_network", "args": { "network_id": "string", - "test_config": "string", - "topology": "string" + "test_config": "string" }, "command": "MOCK_NETWORK network create --network-id {{network_id}} --test-config {{test_config}}" }, @@ -32,7 +31,7 @@ "fresh_state": "bool", "git_commit": "string" }, - "command": "MOCK_NETWORK node start --fresh-state {{fresh_state}} --git-commit {{git_commit}} --network-id {{network_id}} --node-id {{node_id}}" + "command": "MOCK_NETWORK node start {{fresh_state}} --git-commit {{git_commit}} --network-id {{network_id}} --node-id {{node_id}}" }, { "name": "stop_node", diff --git a/src/lib/integration_test_abstract_engine/config.json b/src/lib/integration_test_abstract_engine/config.json index 10ad130694a..c52662a8d64 100644 --- a/src/lib/integration_test_abstract_engine/config.json +++ b/src/lib/integration_test_abstract_engine/config.json @@ -7,21 +7,21 @@ "command": "minimina network create" }, { - "name": "deploy_network", + "name": "start_network", "args": { - "network_id":"string" + "network_id": "string" }, - "command": "minimina network deploy --network-id {{network_id}}" + "command": "minimina network start --network-id {{network_id}}" }, { - "name": "destroy_network", + "name": "delete_network", "args": { - "network_id":"string" + "network_id": "string" }, - "command": "minimina network destroy --network-id {{network_id}}" + "command": "minimina network delete --network-id {{network_id}}" }, { - "name": "get_network_status", + "name": "network_status", "args": { "network_id": "string" }, From fe82d5050fe4157f08e568517c8ea57dad694307 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 16 Aug 2023 04:24:33 +0000 Subject: [PATCH 34/90] (config) Fix return types and flag substitutions --- .../ci_interaction.ml | 169 +++++++++++++----- 1 file changed, 125 insertions(+), 44 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 5033564c37c..3e1482b82ba 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -5,6 +5,12 @@ open Signature_lib open Mina_base open Integration_test_lib +(* [alias] is instantiated when command line args are parsed *) +let alias = ref None + +(* [config_path] is instantiated when command line args are parsed *) +let config_path = ref "" + module Network_config = struct module Cli_inputs = Cli_inputs @@ -72,7 +78,7 @@ module Network_config = struct let testnet_log_filter t = t.config.network_id - (* TODO: double check *) + (* TODO: add config file command call here *) let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) = @@ -305,7 +311,6 @@ module Network_config = struct ; worker_nodes = node.worker_nodes } in - (* NETWORK CONFIG *) { debug_arg = debug ; genesis_keypairs @@ -344,13 +349,32 @@ module Node_type = struct | Archive_node | Block_producer_node | Seed_node - | Snark_worker | Snark_coordinator - [@@deriving eq, yojson] + | Snark_worker + [@@deriving eq, to_yojson] let to_string nt = to_yojson nt |> Yojson.Safe.to_string - let of_string s = of_yojson @@ `List [ `String s ] |> Result.ok_or_failwith + let of_yojson (t : Yojson.Safe.t) = + match t with + | `String nt -> ( + match nt with + | "Archive_node" | "ArchiveNode" | "Archive" -> + Ok Archive_node + | "Block_producer_node" | "BlockProducerNode" | "BlockProducer" -> + Ok Block_producer_node + | "Seed_node" | "SeedNode" | "Seed" -> + Ok Seed_node + | "Snark_coordinator" | "SnarkCoordinator" -> + Ok Snark_coordinator + | "Snark_worker" | "SnarkWorker" -> + Ok Snark_worker + | _ -> + Error (sprintf "Invalid node type: %s" nt) ) + | _ -> + Error (Yojson.Safe.to_string t) + + let of_string s = of_yojson @@ `String s |> Result.ok_or_failwith let is_archive_node = function Archive_node -> true | _ -> false @@ -388,11 +412,11 @@ module Network_deployed = struct exception Invalid_keypair of Yojson.Safe.t type node_info = - { node_id : Node_id.t - ; node_type : Node_type.t + { graphql_uri : string option ; network_id : Network_id.t ; network_keypair : Network_keypair.t option - ; graphql_uri : string option + ; node_id : Node_id.t + ; node_type : Node_type.t } [@@deriving to_yojson] @@ -451,14 +475,12 @@ module Network_deployed = struct let of_yojson = let f network_id accum = function - | `Assoc - [ ( node_id - , `Assoc - [ ("node_type", `String nt) - ; ("private_key", private_key) - ; ("graphql_uri", graphql_uri) - ] ) - ] -> + | ( node_id + , `Assoc + [ ("graphql_uri", graphql_uri) + ; ("node_type", `String nt) + ; ("private_key", private_key) + ] ) -> Core.String.Map.set accum ~key:node_id ~data: { node_id @@ -467,49 +489,60 @@ module Network_deployed = struct ; network_keypair = network_keypair_of_yojson node_id private_key ; graphql_uri = graphql_uri_of_yojson graphql_uri } - | t -> - raise @@ Invalid_entry (Yojson.Safe.to_string t) + | node_id, t -> + raise @@ Invalid_entry (node_id ^ ": " ^ Yojson.Safe.to_string t) in function - | `Assoc [ ("network_id", `String network_id); ("nodes", `List nodes) ] -> + | (`Assoc [ ("network_id", `String network_id); ("node_map", `Assoc nodes) ] : + Yojson.Safe.t ) -> Ok (List.fold nodes ~init:Core.String.Map.empty ~f:(f network_id)) + | `Assoc [ ("network_id", `String network_id) ] -> + (* TODO: remove log *) + [%log' trace (Logger.create ())] "network_id: %s" network_id ; + Ok Core.String.Map.empty | t -> raise @@ Invalid_output (Yojson.Safe.to_string t) end module Node_started = struct - type t = { node_id : Node_id.t } [@@deriving eq, of_yojson] + type t = { network_id : Network_id.t; node_id : Node_id.t } + [@@deriving eq, of_yojson] exception Invalid_response of string end module Node_stopped = struct - type t = { node_id : Node_id.t } [@@deriving eq, of_yojson] + type t = { network_id : Network_id.t; node_id : Node_id.t } + [@@deriving eq, of_yojson] exception Invalid_response of string end module Archive_data_dump = struct - type t = { node_id : Node_id.t; data : string } [@@deriving eq, of_yojson] + type t = { data : string; network_id : Network_id.t; node_id : Node_id.t } + [@@deriving eq, of_yojson] exception Invalid_response of string end module Mina_logs_dump = struct - type t = { node_id : Node_id.t; logs : Node_logs.t } + type t = + { logs : Node_logs.t; network_id : Network_id.t; node_id : Node_id.t } [@@deriving eq, of_yojson] exception Invalid_response of string end module Precomputed_block_dump = struct - type t = { node_id : Node_id.t; blocks : string } [@@deriving eq, of_yojson] + type t = { blocks : string; network_id : Network_id.t; node_id : Node_id.t } + [@@deriving eq, of_yojson] exception Invalid_response of string end module Replayer_run = struct - type t = { node_id : Node_id.t; logs : Node_logs.t } + type t = + { logs : Node_logs.t; network_id : Network_id.t; node_id : Node_id.t } [@@deriving eq, of_yojson] exception Invalid_response of string @@ -578,8 +611,11 @@ end module Config_file = struct type t = { version : int; actions : action list } [@@deriving eq, yojson] - and action = { name : string; args : Yojson.Safe.t; command : string } - [@@deriving eq, yojson] + and action = + { name : string + ; args : Yojson.Safe.t + ; command : string + } exception Invalid_version of Yojson.Safe.t @@ -589,9 +625,11 @@ module Config_file = struct exception Invalid_arg_type_should_be_string of string * Yojson.Safe.t + exception Invalid_list of Yojson.Safe.t list + exception Invalid_command of Yojson.Safe.t - exception Invalid_args_num + exception Invalid_args_num of string * string list * string list exception Missing_arg of string * string @@ -604,6 +642,10 @@ module Config_file = struct | t -> raise @@ Invalid_version t + let action_name action = + let open Yojson.Safe in + Util.member "name" action |> to_string + let action name config = let open Yojson.Safe in match Util.member "actions" config with @@ -628,20 +670,36 @@ module Config_file = struct | name, t -> raise @@ Invalid_arg_type_should_be_string (name, t) ) - let raw_cmd t = - match Yojson.Safe.Util.member "command" t with + let raw_cmd action = + match Yojson.Safe.Util.member "command" action with | `String raw_cmd -> raw_cmd | cmd -> raise @@ Invalid_command cmd + let arg_to_flag s = + "--" ^ String.substr_replace_all s ~pattern:"_" ~with_:"-" + + let validate_cmd_args action = + let open List in + let args = args_of_action action |> map ~f:fst in + let raw_cmd = raw_cmd action in + let contains arg = + String.substr_index raw_cmd ~pattern:(arg_to_flag arg) |> Option.is_some + in + for_all args ~f:contains + let validate_arg_type arg_typ arg_value = String.equal arg_typ @@ Arg_type.arg_type arg_value let validate_args ~args ~action = let arg_list = args_of_action action in let () = - if not List.(length arg_list = length args) then raise @@ Invalid_args_num + let open List in + if not (length arg_list = length args) then + let input_args = map args ~f:fst in + let action_args = map ~f:fst @@ args_of_action action in + raise @@ Invalid_args_num (action_name action, input_args, action_args) in let rec aux = function | [] -> @@ -658,21 +716,41 @@ module Config_file = struct in aux arg_list + let eliminate_bool_arg raw_cmd pattern = + let pattern = " " ^ pattern in + String.substr_replace_all raw_cmd ~pattern ~with_:"" + let rec interpolate_args ~args raw_cmd = match args with | [] -> raw_cmd - | (arg, value) :: args -> - let pattern = sprintf "{{%s}}" arg in - interpolate_args ~args - @@ String.substr_replace_all raw_cmd ~pattern - ~with_:(Yojson.Safe.to_string value |> Util.drop_outer_quotes) + | (arg, value) :: args -> ( + let pattern = sprintf "{{%s}}" arg in + match value with + | `Bool b -> + if b then + interpolate_args ~args + @@ String.substr_replace_all raw_cmd ~pattern + ~with_:(arg_to_flag arg) + else + eliminate_bool_arg raw_cmd pattern |> interpolate_args ~args + | _ -> + interpolate_args ~args + @@ String.substr_replace_all raw_cmd ~pattern + ~with_:(Yojson.Safe.to_string value |> Util.drop_outer_quotes) ) let prog_and_args ~args action = let raw_cmd = raw_cmd action in let cmd_list = interpolate_args ~args raw_cmd |> String.split ~on:' ' in (List.hd_exn cmd_list, List.tl_exn cmd_list) + let evaluate_alias prog = + match !alias with + | Some (alias, value) when String.(alias = prog) -> + value + | _ -> + prog + let run_command ?(suppress_logs = false) ?(timeout_seconds = 1) ?(dir = ".") ~config ~args cmd_name = let config = Yojson.Safe.from_file config in @@ -681,33 +759,36 @@ module Config_file = struct let action_args = args_of_action action in let () = assert (validate_args ~args ~action) in let prog, arg_values = prog_and_args ~args action in + let prog = evaluate_alias prog in let cmd = String.concat ~sep:" " (prog :: arg_values) in let%map output = Util.run_cmd_or_error_timeout ~suppress_logs ~timeout_seconds dir prog arg_values in + let logger = Logger.create () in match output with | Ok output -> if not suppress_logs then - [%log' spam (Logger.create ())] - "Successful command execution\n Command: %s\n Output: %s" cmd output ; + [%log spam] "Successful command execution\n Command: %s\n Output: %s" + cmd output ; Ok output - | _ -> - if not suppress_logs then - [%log' error (Logger.create ())] "Failed to run command: %s" cmd ; + | Error err -> + if not suppress_logs then [%log error] "Failed to run command: %s" cmd ; Error (sprintf "Failed to run command: %s\n\ \ Config version: %d\n\ - \ Expected args types:\n\ + \ Expected arg types:\n\ \ %s\n\ \ Attempted args:\n\ - \ %s\n" + \ %s\n\ + \ Output: %s\n" cmd version ( String.concat ~sep:", " @@ List.map action_args ~f:(fun (name, type_name) -> name ^ ": " ^ type_name ) ) ( String.concat ~sep:", " @@ List.map args ~f:(fun (name, value) -> - name ^ " = " ^ Yojson.Safe.to_string value ) ) ) + name ^ " = " ^ Yojson.Safe.to_string value ) ) + (Error.to_string_hum err) ) end From 966db5e7b7a2d0a351dd15e280cbd179076a1a44 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 16 Aug 2023 04:28:42 +0000 Subject: [PATCH 35/90] (test exec) Use test config to create Network manager --- src/lib/integration_test_abstract_engine/dune | 1 + .../mina_automation.ml | 80 ++++++++----------- src/lib/integration_test_lib/intf.ml | 2 +- 3 files changed, 37 insertions(+), 46 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/dune b/src/lib/integration_test_abstract_engine/dune index 0f7072c8875..98235b4523a 100644 --- a/src/lib/integration_test_abstract_engine/dune +++ b/src/lib/integration_test_abstract_engine/dune @@ -14,6 +14,7 @@ async_unix async_kernel core_kernel + core_unix core cohttp cohttp-async diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index 1def2d691f7..ca2cf8494e1 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -1,4 +1,5 @@ open Core +open Core_unix open Async open Integration_test_lib open Ci_interaction @@ -13,6 +14,7 @@ module Network_manager = struct ; constants : Test_config.constants ; mutable deployed : bool ; genesis_keypairs : Network_keypair.t Core.String.Map.t + ; test_config_path : string } let run_cmd t prog args = Util.run_cmd t.testnet_dir prog args @@ -22,43 +24,18 @@ module Network_manager = struct let run_cmd_or_hard_error t prog args = Util.run_cmd_or_hard_error t.testnet_dir prog args - let create ~logger (network_config : Network_config.t) = + let create ~logger (network_config : Network_config.t) (test_config : Test_config.t) = let open Malleable_error.Let_syntax in - let%bind current_cluster = - (* TODO: replace with Config_file.run_command *) - Util.run_cmd_or_hard_error "/" "kubectl" [ "config"; "current-context" ] - in - [%log info] "Using cluster: %s" current_cluster ; - let%bind all_namespaces_str = - Util.run_cmd_or_hard_error "/" "kubectl" - [ "get"; "namespaces"; "-ojsonpath={.items[*].metadata.name}" ] - in - let all_namespaces = String.split ~on:' ' all_namespaces_str in let%bind () = - if - List.mem all_namespaces network_config.config.network_id - ~equal:String.equal - then - let%bind () = - if network_config.debug_arg then - Deferred.bind ~f:Malleable_error.return - (Util.prompt_continue - "Existing namespace of same name detected, pausing startup. \ - Enter [y/Y] to continue on and remove existing namespace, \ - start clean, and run the test; press Cntrl-C to quit out: " ) - else - Malleable_error.return - @@ [%log info] - "Existing namespace of same name detected; removing to start \ - clean" - in - (* TODO: replace with Config_file.run_command *) - Util.run_cmd_or_hard_error "/" "kubectl" - [ "delete"; "namespace"; network_config.config.network_id ] - >>| Fn.const () - else return () + if network_config.debug_arg then + Deferred.bind ~f:Malleable_error.return + (Util.prompt_continue + "Existing namespace of same name detected, pausing startup. Enter \ + [y/Y] to continue on and remove existing namespace, start clean, \ + and run the test; press Ctrl-C to quit out: " ) + else Malleable_error.return () in - (* TODO: prebuild genesis proof and ledger *) + (* TODO: prebuild genesis proof and ledger and cache for future use *) let testnet_log_filter = Network_config.testnet_log_filter network_config in (* we currently only deploy 1 seed and coordinator per deploy (will be configurable later) *) (* seed node keyname and workload name hardcoded as "seed" *) @@ -66,6 +43,13 @@ module Network_manager = struct network_config.config.config_dir ^/ "/testnets" ^/ network_config.config.network_id in + let open Deferred.Let_syntax in + [%log info] "Making testnet dir %s" testnet_dir ; + let%bind exists = Sys.file_exists_exn testnet_dir in + ignore @@ if exists then Unix.remove testnet_dir else return () ; + let () = mkdir_p testnet_dir in + let network_config_filename = testnet_dir ^/ "network_config.json" in + let test_config_filename = testnet_dir ^/ "test_config.json" in let t = { logger ; network_id = network_config.config.network_id @@ -75,18 +59,17 @@ module Network_manager = struct ; constants = network_config.constants ; deployed = false ; genesis_keypairs = network_config.genesis_keypairs + ; test_config_path = test_config_filename } in - let open Deferred.Let_syntax in - [%log info] "Making testnet dir %s" testnet_dir ; - let%bind () = Unix.mkdir testnet_dir in - let network_config_filename = testnet_dir ^/ "network_config.json" in [%log info] "Writing network configuration into %s" network_config_filename ; Out_channel.with_file ~fail_if_exists:true network_config_filename ~f:(fun ch -> - Network_config.to_yojson network_config - |> Yojson.Safe.to_string - |> Out_channel.output_string ch ) ; + Network_config.to_yojson network_config |> Yojson.Safe.to_channel ch ) ; + [%log info] "Writing test configuration into %s" test_config_filename ; + Out_channel.with_file ~fail_if_exists:true test_config_filename + ~f:(fun ch -> + Test_config.to_yojson test_config |> Yojson.Safe.to_channel ch ) ; [%log info] "Writing out the genesis keys (in case you want to use them manually) to \ testnet dir %s" @@ -102,7 +85,6 @@ module Network_manager = struct in Malleable_error.return t - (* TODO: use output *) let deploy t = let open Network_deployed in let logger = t.logger in @@ -112,7 +94,11 @@ module Network_manager = struct match%map Config_file.run_command ~config:!Abstract_network.config_path - ~args:[] "deploy_network" + ~args: + [ ("network_id", `String t.network_id) + ; ("test_config", `String t.test_config_path) + ] + "create_network" with | Ok output -> output |> Yojson.Safe.from_string |> of_yojson @@ -152,13 +138,17 @@ module Network_manager = struct let destroy t = [%log' info t.logger] "Destroying network" ; if not t.deployed then failwith "network not deployed" ; - let%bind _ = run_cmd_exn t "terraform" [ "destroy"; "-auto-approve" ] in + let%bind _ = + Config_file.run_command ~config:!config_path + ~args:[ ("network_id", `String t.network_id) ] + "delete_network" + in t.deployed <- false ; Deferred.unit let cleanup t = let%bind () = if t.deployed then destroy t else return () in - [%log' info t.logger] "Cleaning up network configuration" ; + [%log' info t.logger] "Cleaning up network directory" ; let%bind () = File_system.remove_dir t.testnet_dir in Deferred.unit diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index 0959dbdf2cf..b462459a279 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -119,7 +119,7 @@ module Engine = struct type t - val create : logger:Logger.t -> Network_config.t -> t Malleable_error.t + val create : logger:Logger.t -> Network_config.t -> Test_config.t -> t Malleable_error.t val deploy : t -> Network.t Malleable_error.t From 6c479eeb1774032a44418d1a7c7d818e04b9b003 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 16 Aug 2023 04:30:08 +0000 Subject: [PATCH 36/90] (test exec) Add mock alias command line arg --- src/app/test_executive/test_executive.ml | 49 +++++-- .../abstract_network.ml | 133 +++--------------- .../ci_interaction.ml | 68 ++++----- .../cli_inputs.ml | 2 - .../config.json | 2 +- .../graphql_polling_log_engine.ml | 4 +- .../mina_automation.ml | 3 +- .../unit_tests.ml | 96 ++++++------- src/lib/integration_test_lib/dsl.ml | 17 +-- src/lib/integration_test_lib/intf.ml | 6 +- .../integration_test_lib/wait_condition.ml | 2 - 11 files changed, 141 insertions(+), 241 deletions(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index 57b431395fa..cd2c148a3f7 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -31,8 +31,14 @@ type inputs = ; archive_image : string option ; debug : bool ; config_path : string option + ; mock_alias : (string * string) option } +let test_name (test : test) + (module Inputs : Integration_test_lib.Intf.Test.Inputs_intf) = + let module Test = (val test) (Inputs) in + Test.test_name + let validate_inputs ~logger inputs (test_config : Test_config.t) : unit Deferred.t = if String.is_empty inputs.mina_image then ( @@ -49,6 +55,15 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : let (Test_inputs_with_cli_inputs ((module Inputs), _)) = inputs.test_inputs in + if + String.(test_name inputs.test (module Inputs) = "mock") + && Option.is_none inputs.mock_alias + && (Option.is_none @@ Sys.getenv "MOCK_NETWORK") + then ( + [%log fatal] + "Must provide either --mock command line arg or set MOCK_NETWORK env \ + var" ; + ignore @@ exit 1 ) ; match Inputs.Engine.name with | "abstract" -> if Option.is_none inputs.config_path then ( @@ -254,11 +269,6 @@ let dispatch_cleanup ~logger ~pause_cleanup_func ~network_cleanup_func cleanup_deferred_ref := Some deferred ; deferred -let test_name (test : test) - (module Inputs : Integration_test_lib.Intf.Test.Inputs_intf) = - let module Test = (val test) (Inputs) in - Test.test_name - let main inputs = let (Test_inputs_with_cli_inputs ((module Test_inputs), cli_inputs)) = inputs.test_inputs @@ -329,7 +339,7 @@ let main inputs = in [%log trace] "initializing network manager" ; let%bind net_manager = - Engine.Network_manager.create ~logger network_config + Engine.Network_manager.create ~logger network_config T.config in net_manager_ref := Some net_manager ; [%log trace] "deploying network" ; @@ -361,7 +371,7 @@ let main inputs = [%log info] "starting the daemons within the pods" ; let start_print (node : Engine.Network.Node.t) = let open Malleable_error.Let_syntax in - [%log info] "starting %s ..." (Engine.Network.Node.id node) ; + [%log info] "starting %s..." (Engine.Network.Node.id node) ; let%bind res = Engine.Network.Node.start ~fresh_state:false node in [%log info] "%s started" (Engine.Network.Node.id node) ; Malleable_error.return res @@ -390,7 +400,7 @@ let main inputs = (* TODO: parallelize (requires accumlative hard errors) *) let%bind () = Malleable_error.List.iter seed_nodes ~f:start_print in let%bind () = - Dsl.wait_for dsl (Dsl.Wait_condition.nodes_to_initialize seed_nodes) + Dsl.(wait_for dsl @@ Wait_condition.nodes_to_initialize seed_nodes) in let%bind () = Malleable_error.List.iter non_seed_pods ~f:start_print in [%log info] "daemons started" ; @@ -431,12 +441,12 @@ let config_path_arg = & opt (some non_dir_file) None & info [ "config-path"; "config" ] ~env ~docv:"MINA_CI_CONFIG_PATH" ~doc) -let alias_arg = +let mock_alias_arg = let doc = "Alias to use for the mock network binary." in let env = Arg.env_var "MOCK_NETWORK" ~doc in Arg.( value - & opt (some @@ pair ~sep:',' string string) None + & opt (some string) None & info [ "mock-network"; "mock"; "alias" ] ~env ~docv:"MOCK_NETWORK" ~doc) let mina_image_arg = @@ -483,7 +493,11 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = Option.iter path ~f:(fun p -> Engine.Network.config_path := p) ; path in - let set_alias alias = Engine.Network.alias := alias in + let set_alias alias = + let alias = Option.map alias ~f:(fun a -> ("MOCK_NETWORK", a)) in + Engine.Network.alias := alias ; + alias + in let test_inputs_with_cli_inputs_arg = let wrap_cli_inputs cli_inputs = Test_inputs_with_cli_inputs ((module Inputs), cli_inputs) @@ -492,15 +506,22 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = in let inputs_term = let cons_inputs test_inputs test archive_image debug mina_image config_path - _ = - { test_inputs; test; mina_image; archive_image; debug; config_path } + mock_alias = + { test_inputs + ; test + ; mina_image + ; archive_image + ; debug + ; config_path + ; mock_alias + } in Term.( const cons_inputs $ test_inputs_with_cli_inputs_arg $ test_arg (module Inputs) $ archive_image_arg $ debug_arg $ mina_image_arg $ (const set_config $ config_path_arg) - $ (const set_alias $ alias_arg)) + $ (const set_alias $ mock_alias_arg)) in (Term.(const start $ inputs_term), info) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index 2ca134bfebe..505fbd4386f 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -29,10 +29,10 @@ module Node = struct unit Malleable_error.t = try let args = - [ ("network_id", `String t.network_id) - ; ("node_id", `String t.node_id) - ; ("fresh_state", `Bool fresh_state) + [ ("fresh_state", `Bool fresh_state) ; ("git_commit", `String git_commit) + ; ("network_id", `String t.network_id) + ; ("node_id", `String t.node_id) ] in let%bind () = @@ -1000,8 +1000,9 @@ module Node = struct in match res with | Ok query_result_obj -> - let had_already_started = query_result_obj.startFilteredLog in - if had_already_started then return (Ok ()) + if query_result_obj.startFilteredLog then ( + [%log trace] "%s's log filter started" @@ id t ; + return (Ok ()) ) else ( [%log error] "Attempted to start structured log collection on $node, but it had \ @@ -1009,6 +1010,11 @@ module Node = struct ~metadata:[ ("node", `String t.node_id) ] ; return (Ok ()) ) | Error e -> + [%log error] "Error encountered while starting $node's log filter: $err" + ~metadata: + [ ("node", `String (id t)) + ; ("err", `String (Error.to_string_hum e)) + ] ; return (Error e) let get_filtered_log_entries ~last_log_index_seen t = @@ -1021,8 +1027,8 @@ module Node = struct exec_graphql_request ~logger:(Logger.null ()) ~retry_delay_sec:10.0 ~node:t ~query_name:"GetFilteredLogEntries" query_obj in - let res = query_result_obj.getFilteredLogEntries in - if res.isCapturing then return res.logMessages + let log_entries = query_result_obj.getFilteredLogEntries in + if log_entries.isCapturing then return log_entries.logMessages else Deferred.Or_error.error_string "Node is not currently capturing structured log messages" @@ -1254,7 +1260,6 @@ let all_nodes { seeds; block_producers; snark_coordinators; archive_nodes; _ } = ] |> Core.String.Map.of_alist_exn -(* TODO: this info is returned from CI *) let all_pods t = List.concat [ Core.String.Map.to_alist t.seeds @@ -1279,113 +1284,9 @@ let genesis_keypairs { genesis_keypairs; _ } = genesis_keypairs let all_node_id t = let pods = all_pods t |> Core.Map.to_alist in - List.fold pods ~init:[] ~f:(fun acc (_, node) -> List.cons node.node_id acc) + List.fold pods ~init:[] ~f:(fun acc (_, node) -> node.node_id :: acc) -(* TODO: remove *) +(* TODO: expand or remove *) let initialize_infra ~logger network = - let open Malleable_error.Let_syntax in - let poll_interval = Time.Span.of_sec 15.0 in - let max_polls = 40 (* 10 mins *) in - let all_pods_set = all_node_id network |> String.Set.of_list in - let kube_get_pods () = - Integration_test_lib.Util.run_cmd_or_error_timeout ~timeout_seconds:60 "/" - "kubectl" - [ "-n" - ; "namespace" - ; "get" - ; "pods" - ; "-ojsonpath={range \ - .items[*]}{.metadata.name}{':'}{.status.phase}{'\\n'}{end}" - ] - in - let parse_pod_statuses result_str = - result_str |> String.split_lines - |> List.map ~f:(fun line -> - let parts = String.split line ~on:':' in - assert (Mina_stdlib.List.Length.Compare.(parts = 2)) ; - (List.nth_exn parts 0, List.nth_exn parts 1) ) - |> List.filter ~f:(fun (pod_name, _) -> - String.Set.mem all_pods_set pod_name ) - (* this filters out the archive bootstrap pods, since they aren't in all_pods_set. in fact the bootstrap pods aren't tracked at all in the framework *) - |> String.Map.of_alist_exn - in - let rec poll n = - [%log debug] "Checking kubernetes pod statuses, n=%d" n ; - let is_successful_pod_status status = String.equal "Running" status in - match%bind Deferred.bind ~f:Malleable_error.return (kube_get_pods ()) with - | Ok str -> - let pod_statuses = parse_pod_statuses str in - [%log debug] "pod_statuses: \n %s" - ( String.Map.to_alist pod_statuses - |> List.map ~f:(fun (key, data) -> key ^ ": " ^ data ^ "\n") - |> String.concat ) ; - [%log debug] "all_pods: \n %s" - (String.Set.elements all_pods_set |> String.concat ~sep:", ") ; - let all_pods_are_present = - List.for_all (String.Set.elements all_pods_set) ~f:(fun pod_id -> - String.Map.mem pod_statuses pod_id ) - in - let any_pods_are_not_running = - (* there could be duplicate keys... *) - List.exists - (String.Map.data pod_statuses) - ~f:(Fn.compose not is_successful_pod_status) - in - if not all_pods_are_present then ( - let present_pods = String.Map.keys pod_statuses in - [%log fatal] - "Not all pods were found when querying namespace; this indicates a \ - deployment error. Refusing to continue. \n\ - Expected pods: [%s]. \n\ - Present pods: [%s]" - (String.Set.elements all_pods_set |> String.concat ~sep:"; ") - (present_pods |> String.concat ~sep:"; ") ; - Malleable_error.hard_error_string ~exit_code:5 - "Some pods were not found in namespace." ) - else if any_pods_are_not_running then - let failed_pod_statuses = - List.filter (String.Map.to_alist pod_statuses) - ~f:(fun (_, status) -> not (is_successful_pod_status status)) - in - if n > 0 then ( - [%log debug] "Got bad pod statuses, polling again ($failed_statuses" - ~metadata: - [ ( "failed_statuses" - , `Assoc - (List.Assoc.map failed_pod_statuses ~f:(fun v -> - `String v ) ) ) - ] ; - let%bind () = - after poll_interval |> Deferred.bind ~f:Malleable_error.return - in - poll (n - 1) ) - else ( - [%log fatal] - "Got bad pod statuses, not all pods were assigned to nodes and \ - ready in time. pod statuses: ($failed_statuses" - ~metadata: - [ ( "failed_statuses" - , `Assoc - (List.Assoc.map failed_pod_statuses ~f:(fun v -> - `String v ) ) ) - ] ; - Malleable_error.hard_error_string ~exit_code:4 - "Some pods either were not assigned to nodes or did not deploy \ - properly." ) - else return () - | Error _ -> - [%log debug] "`kubectl get pods` timed out, polling again" ; - let%bind () = - after poll_interval |> Deferred.bind ~f:Malleable_error.return - in - poll n - in - [%log info] "Waiting for pods to be assigned nodes and become ready" ; - let res = poll max_polls in - match%bind.Deferred res with - | Error _ -> - [%log error] "Not all pods were assigned nodes, cannot proceed!" ; - res - | Ok _ -> - [%log info] "Pods assigned to nodes" ; - res + [%log trace] "initialize_infra: %s" network.network_id ; + Malleable_error.return () diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 3e1482b82ba..21ad41b206e 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -474,13 +474,15 @@ module Network_deployed = struct raise @@ Invalid_argument (Yojson.Safe.to_string uri) let of_yojson = - let f network_id accum = function - | ( node_id - , `Assoc - [ ("graphql_uri", graphql_uri) - ; ("node_type", `String nt) - ; ("private_key", private_key) - ] ) -> + let f network_id accum : Yojson.Safe.t -> t = function + | `Assoc + [ ( node_id + , `Assoc + [ ("graphql_uri", graphql_uri) + ; ("node_type", `String nt) + ; ("private_key", private_key) + ] ) + ] -> Core.String.Map.set accum ~key:node_id ~data: { node_id @@ -489,18 +491,18 @@ module Network_deployed = struct ; network_keypair = network_keypair_of_yojson node_id private_key ; graphql_uri = graphql_uri_of_yojson graphql_uri } - | node_id, t -> - raise @@ Invalid_entry (node_id ^ ": " ^ Yojson.Safe.to_string t) + | t -> + raise @@ Invalid_entry (Yojson.Safe.to_string t) in function - | (`Assoc [ ("network_id", `String network_id); ("node_map", `Assoc nodes) ] : + | (`Assoc [ ("network_id", `String network_id); ("nodes", `List nodes) ] : Yojson.Safe.t ) -> Ok (List.fold nodes ~init:Core.String.Map.empty ~f:(f network_id)) - | `Assoc [ ("network_id", `String network_id) ] -> - (* TODO: remove log *) - [%log' trace (Logger.create ())] "network_id: %s" network_id ; + | `Assoc [ ("network_id", `String _network_id) ] -> + (* TODO: remove *) Ok Core.String.Map.empty | t -> + print_endline @@ Yojson.Safe.pretty_to_string t ; raise @@ Invalid_output (Yojson.Safe.to_string t) end @@ -611,11 +613,7 @@ end module Config_file = struct type t = { version : int; actions : action list } [@@deriving eq, yojson] - and action = - { name : string - ; args : Yojson.Safe.t - ; command : string - } + and action = { name : string; args : Yojson.Safe.t; command : string } exception Invalid_version of Yojson.Safe.t @@ -677,17 +675,7 @@ module Config_file = struct | cmd -> raise @@ Invalid_command cmd - let arg_to_flag s = - "--" ^ String.substr_replace_all s ~pattern:"_" ~with_:"-" - - let validate_cmd_args action = - let open List in - let args = args_of_action action |> map ~f:fst in - let raw_cmd = raw_cmd action in - let contains arg = - String.substr_index raw_cmd ~pattern:(arg_to_flag arg) |> Option.is_some - in - for_all args ~f:contains + let arg_to_flag s = "--" ^ String.substr_replace_all s ~pattern:"_" ~with_:"-" let validate_arg_type arg_typ arg_value = String.equal arg_typ @@ Arg_type.arg_type arg_value @@ -725,19 +713,19 @@ module Config_file = struct | [] -> raw_cmd | (arg, value) :: args -> ( - let pattern = sprintf "{{%s}}" arg in - match value with - | `Bool b -> - if b then - interpolate_args ~args - @@ String.substr_replace_all raw_cmd ~pattern - ~with_:(arg_to_flag arg) - else - eliminate_bool_arg raw_cmd pattern |> interpolate_args ~args - | _ -> + let pattern = sprintf "{{%s}}" arg in + match value with + | `Bool b -> + if b then interpolate_args ~args @@ String.substr_replace_all raw_cmd ~pattern - ~with_:(Yojson.Safe.to_string value |> Util.drop_outer_quotes) ) + ~with_:(arg_to_flag arg) + else eliminate_bool_arg raw_cmd pattern |> interpolate_args ~args + | _ -> + interpolate_args ~args + @@ String.substr_replace_all raw_cmd ~pattern + ~with_:(Yojson.Safe.to_string value |> Util.drop_outer_quotes) + ) let prog_and_args ~args action = let raw_cmd = raw_cmd action in diff --git a/src/lib/integration_test_abstract_engine/cli_inputs.ml b/src/lib/integration_test_abstract_engine/cli_inputs.ml index 5135757dcac..89a626a3990 100644 --- a/src/lib/integration_test_abstract_engine/cli_inputs.ml +++ b/src/lib/integration_test_abstract_engine/cli_inputs.ml @@ -1,7 +1,5 @@ open Cmdliner -(* TODO: finalize this *) - type t = { mina_automation_location : string } let term = diff --git a/src/lib/integration_test_abstract_engine/config.json b/src/lib/integration_test_abstract_engine/config.json index c52662a8d64..06951e3f21b 100644 --- a/src/lib/integration_test_abstract_engine/config.json +++ b/src/lib/integration_test_abstract_engine/config.json @@ -35,7 +35,7 @@ "fresh_state":"bool", "git_commit":"string" }, - "command": "minimina node start --network-id {{network_id}} --node-id {{node_id}} --fresh-state {{fresh_state}} --git-commit {{git_commit}}" + "command": "minimina node start --network-id {{network_id}} --node-id {{node_id}} {{fresh_state}} --git-commit {{git_commit}}" }, { "name": "stop_node", diff --git a/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml b/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml index 4e52efc6793..3f56c30febf 100644 --- a/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml +++ b/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml @@ -12,12 +12,11 @@ let log_filter_of_event_type ev_existential = let (module Ty) = event_type_module ev_type in match Ty.parse with | From_error_log _ -> - [] (* TODO: Do we need this? *) + [] | From_daemon_log (struct_id, _) -> [ Structured_log_events.string_of_id struct_id ] | From_puppeteer_log _ -> [] -(* TODO: Do we need this? *) let all_event_types_log_filter = List.bind ~f:log_filter_of_event_type Event_type.all_event_types @@ -66,6 +65,7 @@ let rec poll_get_filtered_log_entries_node ~logger ~event_writer Array.iter log_entries ~f:(fun log_entry -> match parse_event_from_log_entry ~logger log_entry with | Ok a -> + [%log debug] "Parsed %s's log entry" @@ Node.id node ; Pipe.write_without_pushback_if_open event_writer (node, a) | Error e -> [%log warn] "Error parsing log $error" diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index ca2cf8494e1..9e31a7e17fe 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -24,7 +24,8 @@ module Network_manager = struct let run_cmd_or_hard_error t prog args = Util.run_cmd_or_hard_error t.testnet_dir prog args - let create ~logger (network_config : Network_config.t) (test_config : Test_config.t) = + let create ~logger (network_config : Network_config.t) + (test_config : Test_config.t) = let open Malleable_error.Let_syntax in let%bind () = if network_config.debug_arg then diff --git a/src/lib/integration_test_abstract_engine/unit_tests.ml b/src/lib/integration_test_abstract_engine/unit_tests.ml index 0f27d783c5a..685d7ac56d8 100644 --- a/src/lib/integration_test_abstract_engine/unit_tests.ml +++ b/src/lib/integration_test_abstract_engine/unit_tests.ml @@ -17,7 +17,7 @@ module Test_values = struct let start_node_raw_cmd = "minimina node start --network-id {{network_id}} --node-id {{node_id}} \ - --fresh-state {{fresh_state}} --git-commit {{git_commit}}" + {{fresh_state}} --git-commit {{git_commit}}" let deploy_network_action = sprintf @@ -27,15 +27,6 @@ module Test_values = struct "args": { "network_id": "string" }, - "returns": [ - { - "graphql_uri" : "string", - "network_id" : "string", - "network_keypair" : "string", - "node_id" : "string", - "node_type" : "string" - } - ], "command": "%s" } |} @@ -57,17 +48,17 @@ module Test_values = struct let start_node_action = sprintf {| - { - "name": "start_node", - "args": { - "network_id": "string", - "node_id": "string", - "fresh_state": "bool", - "git_commit": "string" - }, - "command": "%s" - } - |} + { + "name": "start_node", + "args": { + "network_id": "string", + "node_id": "string", + "fresh_state": "bool", + "git_commit": "string" + }, + "command": "%s" + } + |} start_node_raw_cmd |> String.strip @@ -118,8 +109,6 @@ module Config_tests = struct in assert (validate_args ~args ~action) - let%test_unit "TODO: validate_cmd_args" = assert false - let%test_unit "interpolate string args" = let res = interpolate_args @@ -151,11 +140,14 @@ module Config_tests = struct let%test_unit "prog and args" = let action = Yojson.Safe.from_string start_node_action in + let node_id = "node0" in + let network_id = "network0" in + let git_commit = "0123456abcdef" in let args = - [ ("network_id", `String "network0") - ; ("node_id", `String "node0") - ; ("fresh_state", `Bool true) - ; ("git_commit", `String "0123456abcdef") + [ ("fresh_state", `Bool true) + ; ("git_commit", `String git_commit) + ; ("network_id", `String network_id) + ; ("node_id", `String node_id) ] in let prog, res_args = prog_and_args ~args action in @@ -164,13 +156,12 @@ module Config_tests = struct ; "node" ; "start" ; "--network-id" - ; "network0" + ; network_id ; "--node-id" - ; "node0" + ; node_id ; "--fresh-state" - ; "true" ; "--git-commit" - ; "0123456abcdef" + ; git_commit ] in assert (prog = List.hd_exn expect) ; @@ -186,7 +177,10 @@ module Run_command_tests = struct let%test_unit "run command arg type validation failure" = let arg, value = ("msg", `Int 42) in try - ignore @@ run_command ~config:config_file ~args:[ (arg, value) ] "echo" + ignore + @@ run_command ~suppress_logs:true ~config:config_file + ~args:[ (arg, value) ] + "echo" with Invalid_arg_type (arg_name, arg_value, arg_type) -> assert ( arg_name = "msg" && arg_type = Arg_type.string @@ -195,20 +189,20 @@ module Run_command_tests = struct let%test_unit "run command arg number failure" = try ignore - @@ run_command ~config:config_file - ~args:[ ("msg", `String "hello"); ("num", `Int 42) ] + @@ run_command ~suppress_logs:true ~config:config_file + ~args:[ ("msg", `String "hello") ] "echo" with Invalid_args_num (action_name, input_args, action_args) -> let open String in let input_args = concat ~sep:", " input_args in let action_args = concat ~sep:", " action_args in - printf "Action: %s\n Input args: %s\n Expected args: %s\n" action_name + printf "Action: %s\n Input args: %s\n Expected args: %s\n" action_name input_args action_args let%test_unit "run command missing arg failure" = try ignore - @@ run_command ~config:config_file + @@ run_command ~suppress_logs:true ~config:config_file ~args:[ ("msg0", `String "hello") ] "echo" with Missing_arg (arg_name, arg_type) -> @@ -316,33 +310,33 @@ module Parse_output_tests = struct { "network_id":"network0", "nodes": [ { "node0": { - "node_type":"Archive_node", - "private_key":null, - "graphql_uri":"gql_archive" + "graphql_uri": "gql_archive", + "node_type": "Archive_node", + "private_key": null } }, { "node1": { - "node_type":"Block_producer_node", - "private_key":null, - "graphql_uri":"gql_bp" + "graphql_uri": "gql_bp", + "node_type": "Block_producer_node", + "private_key": null } }, { "node2": { - "node_type":"Seed_node", - "private_key":"EKEQpDAjj7dP3j7fQy4qBU7Kxns85wwq5xMn4zxdyQm83pEWzQ62", - "graphql_uri":"gql_seed" + "graphql_uri": "gql_seed", + "node_type": "Seed_node", + "private_key": "EKEQpDAjj7dP3j7fQy4qBU7Kxns85wwq5xMn4zxdyQm83pEWzQ62" } }, { "node3": { - "node_type":"Snark_worker", - "private_key":null, - "graphql_uri":"gql_snark" + "graphql_uri": "gql_snark", + "node_type": "Snark_worker", + "private_key": null } }, { "node4": { - "node_type":"Snark_coordinator", - "private_key":null, - "graphql_uri":null + "graphql_uri": null, + "node_type": "Snark_coordinator", + "private_key": null } } ] diff --git a/src/lib/integration_test_lib/dsl.ml b/src/lib/integration_test_lib/dsl.ml index 149c1abf758..c7f5bbb707c 100644 --- a/src/lib/integration_test_lib/dsl.ml +++ b/src/lib/integration_test_lib/dsl.ml @@ -107,12 +107,11 @@ module Make (Engine : Intf.Engine.S) () : let wait_for t condition = let open Wait_condition in + let open Network_time_span in let constants = Engine.Network.constants t.network in - let soft_timeout = - Network_time_span.to_span condition.soft_timeout ~constants - in - let hard_timeout = - Network_time_span.to_span condition.hard_timeout ~constants + let soft_timeout, hard_timeout = + ( to_span condition.soft_timeout ~constants + , to_span condition.hard_timeout ~constants ) in let start_time = Time.now () in [%log' info t.logger] @@ -120,12 +119,8 @@ module Make (Engine : Intf.Engine.S) () : $hard_timeout)" condition.description ~metadata: - [ ( "soft_timeout" - , `String - (Network_time_span.to_string ~constants condition.soft_timeout) ) - ; ( "hard_timeout" - , `String - (Network_time_span.to_string ~constants condition.hard_timeout) ) + [ ("soft_timeout", `String (to_string ~constants condition.soft_timeout)) + ; ("hard_timeout", `String (to_string ~constants condition.hard_timeout)) ] ; let%bind result = match condition.predicate with diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index b462459a279..37caf38e947 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -119,7 +119,11 @@ module Engine = struct type t - val create : logger:Logger.t -> Network_config.t -> Test_config.t -> t Malleable_error.t + val create : + logger:Logger.t + -> Network_config.t + -> Test_config.t + -> t Malleable_error.t val deploy : t -> Network.t Malleable_error.t diff --git a/src/lib/integration_test_lib/wait_condition.ml b/src/lib/integration_test_lib/wait_condition.ml index 110b2ee38cf..37b54994bfc 100644 --- a/src/lib/integration_test_lib/wait_condition.ml +++ b/src/lib/integration_test_lib/wait_condition.ml @@ -70,7 +70,6 @@ struct let wait_condition_id t = t.id let nodes_to_initialize nodes = - let open Network_state in network_state ~id:Nodes_to_initialize ~description: ( nodes |> List.map ~f:Node.id |> String.concat ~sep:", " @@ -85,7 +84,6 @@ struct let node_to_initialize node = nodes_to_initialize [ node ] - (* let blocks_produced ?(active_stake_percentage = 1.0) n = *) let blocks_to_be_produced n = let init state = Predicate_continuation state.blocks_generated in let check init_blocks_generated state = From debdf7c4857bb08d64b8601566882aa9770e6c2c Mon Sep 17 00:00:00 2001 From: Quantifier Date: Mon, 21 Aug 2023 17:07:27 +0000 Subject: [PATCH 37/90] (unit test) Update network config --- .../unit_tests.ml | 148 +++++++++++------- 1 file changed, 93 insertions(+), 55 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/unit_tests.ml b/src/lib/integration_test_abstract_engine/unit_tests.ml index 685d7ac56d8..5f9a1d3d9ad 100644 --- a/src/lib/integration_test_abstract_engine/unit_tests.ml +++ b/src/lib/integration_test_abstract_engine/unit_tests.ml @@ -72,6 +72,41 @@ module Test_values = struct |} deploy_network_action start_node_action |> Yojson.Safe.from_string + + let network_config : Network_config.config = + let network_id = "network0" in + let config_dir = "." in + let deploy_graphql_ingress = true in + let mina_image = "mina" in + let mina_agent_image = "agent" in + let mina_bots_image = "bots" in + let mina_points_image = "points" in + let mina_archive_image = "archive" in + let runtime_config = `Null in + let block_producer_configs = [] in + let log_precomputed_blocks = false in + let archive_node_count = 0 in + let mina_archive_schema = "schema" in + let mina_archive_schema_aux_files = [] in + let snark_coordinator_config = None in + let snark_worker_fee = "0.01" in + { network_id + ; config_dir + ; deploy_graphql_ingress + ; mina_image + ; mina_agent_image + ; mina_bots_image + ; mina_points_image + ; mina_archive_image + ; runtime_config + ; block_producer_configs + ; log_precomputed_blocks + ; archive_node_count + ; mina_archive_schema + ; mina_archive_schema_aux_files + ; snark_coordinator_config + ; snark_worker_fee + } end open Test_values @@ -118,26 +153,6 @@ module Config_tests = struct let expect = {|minimina network deploy --network-id network0|} in assert (res = expect) - (* let%test_unit "command returns" = - let returns = - returns_of_action @@ Yojson.Safe.from_string deploy_network_action - in - let[@warning "-8"] (Record record) = returns in - let expect = - [ ("graphql_uri", `Null) - ; ("network_id", `String "network0") - ; ( "network_keypair" - , `String "EKEQpDAjj7dP3j7fQy4qBU7Kxns85wwq5xMn4zxdyQm83pEWzQ62" ) - ; ("node_id", `String "node0") - ; ("node_type", `String "Archive_node") - ] - in - let ( = ) = - List.equal (fun (a, b) (c, d) -> - String.equal a c && Yojson.Safe.equal b d ) - in - assert (record = expect) *) - let%test_unit "prog and args" = let action = Yojson.Safe.from_string start_node_action in let node_id = "node0" in @@ -247,57 +262,80 @@ module Request_tests = struct ; genesis = Genesis_constants.compiled } in - let network_config : Network_config.t' = - { debug_arg = true; genesis_keypairs = Core.String.Map.empty; constants } + let config = Test_values.network_config in + let network_config : Network_config.t = + { debug_arg = true + ; genesis_keypairs = Core.String.Map.empty + ; constants + ; config + } in + let network_config = Network_config.to_yojson network_config in let expect = {| - { "debug_arg":true, - "genesis_keypairs":{}, - "constants":{ - "constraints":{ - "sub_windows_per_window":11, - "ledger_depth":35, - "work_delay":2, - "block_window_duration_ms":180000, - "transaction_capacity_log_2":7, - "pending_coinbase_depth":5, - "coinbase_amount":"720000000000", - "supercharged_coinbase_factor":1, - "account_creation_fee":"1", - "fork":null + { "debug_arg": true, + "genesis_keypairs": {}, + "constants": { + "constraints": { + "sub_windows_per_window": 11, + "ledger_depth": 35, + "work_delay": 2, + "block_window_duration_ms": 180000, + "transaction_capacity_log_2": 7, + "pending_coinbase_depth": 5, + "coinbase_amount": "720000000000", + "supercharged_coinbase_factor": 1, + "account_creation_fee": "1", + "fork": null }, - "genesis":{ - "protocol":{ - "k":290, - "slots_per_epoch":7140, - "slots_per_sub_window":7, - "delta":0, - "genesis_state_timestamp":"2020-09-16 10:15:00.000000Z" + "genesis": { + "protocol": { + "k": 290, + "slots_per_epoch": 7140, + "slots_per_sub_window": 7, + "delta": 0, + "genesis_state_timestamp": "2020-09-16 10:15:00.000000Z" }, - "txpool_max_size":3000, - "num_accounts":null, - "zkapp_proof_update_cost":10.26, - "zkapp_signed_single_update_cost":9.140000000000001, - "zkapp_signed_pair_update_cost":10.08, - "zkapp_transaction_cost_limit":69.45, - "max_event_elements":100, - "max_action_elements":100 + "txpool_max_size": 3000, + "num_accounts": null, + "zkapp_proof_update_cost": 10.26, + "zkapp_signed_single_update_cost": 9.140000000000001, + "zkapp_signed_pair_update_cost": 10.08, + "zkapp_transaction_cost_limit": 69.45, + "max_event_elements": 100, + "max_action_elements": 100 } + }, + "config": { + "network_id": "network0", + "config_dir": ".", + "deploy_graphql_ingress": true, + "mina_image": "mina", + "mina_agent_image": "agent", + "mina_bots_image": "bots", + "mina_points_image": "points", + "mina_archive_image": "archive", + "runtime_config": "null", + "block_producer_configs": [], + "log_precomputed_blocks": false, + "archive_node_count": 0, + "mina_archive_schema": "schema", + "mina_archive_schema_aux_files": [], + "snark_coordinator_config": null, + "snark_worker_fee": "0.01" } } |} |> Yojson.Safe.from_string in - assert ( - Yojson.Safe.equal expect @@ Network_config.t'_to_yojson network_config ) + assert (Yojson.Safe.equal expect network_config) end module Parse_output_tests = struct let%test_unit "parse network created" = let open Network_created in let result = - {|{"network_id":"network0"}|} |> Yojson.Safe.from_string |> of_yojson + {|{"network_id": "network0"}|} |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith in assert (equal result { network_id = "network0" }) @@ -307,7 +345,7 @@ module Parse_output_tests = struct let open Network_deployed in let result = {| - { "network_id":"network0", + { "network_id": "network0", "nodes": [ { "node0": { "graphql_uri": "gql_archive", From e3b9daebd404fc075e2881053b0087aaf3c78ce5 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 22 Aug 2023 05:05:26 +0000 Subject: [PATCH 38/90] (config) Replace temp type t' for type t --- .../ci_interaction.ml | 30 +++++-------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 21ad41b206e..9f6edec0f07 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -43,7 +43,6 @@ module Network_config = struct } [@@deriving to_yojson] - (* TODO: replace with t' *) type t = { debug_arg : bool ; genesis_keypairs : @@ -60,22 +59,6 @@ module Network_config = struct } [@@deriving to_yojson] - (* TODO: remove *) - type t' = - { debug_arg : bool - ; genesis_keypairs : - (Network_keypair.t Core.String.Map.t - [@to_yojson - fun map -> - `Assoc - (Core.Map.fold_right ~init:[] - ~f:(fun ~key:k ~data:v accum -> - (k, Network_keypair.to_yojson v) :: accum ) - map )] ) - ; constants : Test_config.constants - } - [@@deriving to_yojson] - let testnet_log_filter t = t.config.network_id (* TODO: add config file command call here *) @@ -211,10 +194,8 @@ module Network_config = struct Some { base = Accounts - (let tuplist = String.Map.data genesis_ledger_accounts in - List.map tuplist ~f:(fun tup -> - let acct, _ = tup in - acct ) ) + ( String.Map.data genesis_ledger_accounts + |> List.map ~f:(fun tup -> fst tup) ) ; add_genesis_winner = None ; num_accounts = None ; balances = [] @@ -277,6 +258,7 @@ module Network_config = struct ; mina_archive_base_url ^ "zkapp_tables.sql" ] in + let before = Time.now () in let genesis_keypairs = String.Map.of_alist_exn (List.map (String.Map.to_alist genesis_ledger_accounts) @@ -284,6 +266,9 @@ module Network_config = struct let kp_name, (_, (pk, sk)) = element in (kp_name, mk_net_keypair kp_name (pk, sk)) ) ) in + [%log trace] "Genesis keypairs (%d): %s" + (String.Map.length genesis_keypairs) + Time.(abs_diff before @@ now () |> Span.to_string) ; let snark_coordinator_config = match snark_coordinator with | None -> @@ -495,8 +480,7 @@ module Network_deployed = struct raise @@ Invalid_entry (Yojson.Safe.to_string t) in function - | (`Assoc [ ("network_id", `String network_id); ("nodes", `List nodes) ] : - Yojson.Safe.t ) -> + | `Assoc [ ("network_id", `String network_id); ("nodes", `List nodes) ] -> Ok (List.fold nodes ~init:Core.String.Map.empty ~f:(f network_id)) | `Assoc [ ("network_id", `String _network_id) ] -> (* TODO: remove *) From 6cdae588f99316a5584c6c98562d56306c5c5d38 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 23 Aug 2023 19:49:40 +0000 Subject: [PATCH 39/90] (git) Add automatation/testnets/ --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 152a4660560..5192525b3b6 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,7 @@ automation/*-accounts.csv automation/block_tree.gv.png automation/gcloud-keyfile.json automation/services/watchdog/check_libp2p/check_libp2p +automation/testnets/ *.backup From cf1eca8fe95f69d24bee5aca8eb31f63937b72bb Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sun, 27 Aug 2023 17:48:50 +0000 Subject: [PATCH 40/90] (test exec) Remove unused abstract engine terraform --- .../terraform.ml | 111 ------------------ 1 file changed, 111 deletions(-) delete mode 100644 src/lib/integration_test_abstract_engine/terraform.ml diff --git a/src/lib/integration_test_abstract_engine/terraform.ml b/src/lib/integration_test_abstract_engine/terraform.ml deleted file mode 100644 index 02eea743983..00000000000 --- a/src/lib/integration_test_abstract_engine/terraform.ml +++ /dev/null @@ -1,111 +0,0 @@ -open Core - -(* this is only a partial (minimum required) implementation of the terraform json spec *) -let cons (type a) (key : string) (body : a) (to_yojson : a -> Yojson.Safe.t) = - `Assoc [ (key, to_yojson body) ] - -module Backend = struct - module Local = struct - type t = { path : string } [@@deriving to_yojson] - end - - type t = Local of Local.t - - let to_yojson = function Local x -> cons "local" x Local.to_yojson -end - -module Block = struct - module Terraform = struct - type t = { required_version : string; backend : Backend.t } - [@@deriving to_yojson] - end - - (* should probably leave these untyped, but this covers basic keys for both google and aws *) - module Provider = struct - type t = - { provider : string - ; region : string - ; alias : string option - ; project : string option - ; zone : string option - } - [@@deriving to_yojson] - - let to_yojson { provider; region; alias; project; zone } = - cons provider () (fun () -> - let open Option.Let_syntax in - let field k v = (k, `String v) in - let fields = - [ Some (field "region" region) - ; alias >>| field "alias" - ; project >>| field "project" - ; zone >>| field "zone" - ] - in - `Assoc (List.filter_map fields ~f:Fn.id) ) - end - - module Module = struct - type t = - { local_name : string - ; providers : (string * string) list - ; source : string - ; args : (string * Yojson.Safe.t) list - } - - let to_yojson { local_name; providers; source; args } = - cons local_name () (fun () -> - let const_fields = - [ ( "providers" - , `Assoc (List.map providers ~f:(fun (k, v) -> (k, `String v))) ) - ; ("source", `String source) - ] - in - `Assoc (const_fields @ args) ) - end - - module Data = struct - type t = - { data_source : string - ; local_name : string - ; args : (string * Yojson.Safe.t) list - } - - let to_yojson { data_source; local_name; args } = - cons data_source () (fun () -> cons local_name () (fun () -> `Assoc args)) - end - - module Resource = struct - type t = - { type_ : string - ; local_name : string - ; args : (string * Yojson.Safe.t) list - } - - let to_yojson { type_; local_name; args } = - cons type_ () (fun () -> cons local_name () (fun () -> `Assoc args)) - end - - type t = - | Terraform of Terraform.t - | Provider of Provider.t - | Module of Module.t - | Data of Data.t - | Resource of Resource.t - - let to_yojson = function - | Terraform x -> - cons "terraform" x Terraform.to_yojson - | Provider x -> - cons "provider" x Provider.to_yojson - | Module x -> - cons "module" x Module.to_yojson - | Data x -> - cons "data" x Data.to_yojson - | Resource x -> - cons "resource" x Resource.to_yojson -end - -type t = Block.t list [@@deriving to_yojson] - -let to_string = Fn.compose Yojson.Safe.pretty_to_string to_yojson From c6e08a4b5132e90e8880733add370187eee8c164 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 29 Aug 2023 16:42:33 +0000 Subject: [PATCH 41/90] (intg test) Add runtime config and topology extraction to Test_config.t for abstract engine --- src/lib/integration_test_lib/test_config.ml | 194 +++++++++++++++++++- 1 file changed, 192 insertions(+), 2 deletions(-) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 30d2328f58a..6962861a031 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -57,6 +57,7 @@ module Test_Account = struct type t = { account_name : string ; balance : string + ; pk : string ; timing : Mina_base.Account_timing.t } [@@deriving to_yojson] @@ -71,15 +72,27 @@ module Epoch_data = struct type t = { staking : Data.t; next : Data.t option } end +module Archive_node = struct + type t = { node_name : string; account_name : string } [@@deriving to_yojson] +end + module Block_producer_node = struct type t = { node_name : string; account_name : string } [@@deriving to_yojson] end +module Seed_node = struct + type t = { node_name : string; account_name : string } [@@deriving to_yojson] +end + module Snark_coordinator_node = struct type t = { node_name : string; account_name : string; worker_nodes : int } [@@deriving to_yojson] end +module Snark_worker_node = struct + type t = { node_name : string; account_name : string } [@@deriving to_yojson] +end + type constants = { constraints : Genesis_constants.Constraint_constants.t ; genesis : Genesis_constants.t @@ -90,10 +103,12 @@ type t = { requires_graphql : bool ; genesis_ledger : Test_Account.t list ; epoch_data : Epoch_data.t option + ; archive_nodes : Archive_node.t list ; block_producers : Block_producer_node.t list + ; seed_nodes : Seed_node.t list ; snark_coordinator : Snark_coordinator_node.t option + ; snark_workers : Snark_worker_node.t list ; snark_worker_fee : string - ; num_archive_nodes : int ; log_precomputed_blocks : bool ; proof_config : Runtime_config.Proof_keys.t ; k : int @@ -104,6 +119,32 @@ type t = } [@@deriving to_yojson] +module Node_role = struct + type t = + | Archive_node + | Block_producer + | Seed_node + | Snark_coordinator + | Snark_worker + + let to_yojson = function + | Archive_node -> + `String "Archive_node" + | Block_producer -> + `String "Block_producer" + | Seed_node -> + `String "Seed_node" + | Snark_coordinator -> + `String "Snark_coordinator" + | Snark_worker -> + `String "Snark_worker" +end + +type topology = topology_node list + +and topology_node = { alias : string; pk : string; role : Node_role.t } +[@@deriving to_yojson] + let proof_config_default : Runtime_config.Proof_keys.t = { level = Some Full ; sub_windows_per_window = None @@ -123,10 +164,12 @@ let default = (* require_graphql maybe should just be phased out, because it always needs to be enable. Now with the graphql polling engine, everything will definitely fail if graphql is not enabled. But even before that, most tests relied on some sort of graphql interaction *) ; genesis_ledger = [] ; epoch_data = None + ; archive_nodes = [] ; block_producers = [] + ; seed_nodes = [] ; snark_coordinator = None + ; snark_workers = [] ; snark_worker_fee = "0.025" - ; num_archive_nodes = 0 ; log_precomputed_blocks = false ; proof_config = proof_config_default ; k = 20 @@ -187,3 +230,150 @@ let transactions_needed_for_ledger_proofs ?(num_proofs = 1) config = let transactions_per_block = transaction_capacity config in (blocks_for_first_ledger_proof config * transactions_per_block) + (transactions_per_block * (num_proofs - 1)) + +let runtime_config_of_test_config t = + let convert : + Mina_base.Account_timing.t + -> Runtime_config.Accounts.Single.Timed.t option = function + | Untimed -> + None + | Timed timing -> + Some + { initial_minimum_balance = timing.initial_minimum_balance + ; cliff_time = timing.cliff_time + ; cliff_amount = timing.cliff_amount + ; vesting_period = timing.vesting_period + ; vesting_increment = timing.vesting_increment + } + in + let open Yojson.Safe in + let timed_opt_to_yojson : Runtime_config.Accounts.Single.Timed.t option -> t = + function + | None -> + `Null + | Some timed -> + Runtime_config.Accounts.Single.Timed.to_yojson timed + in + let accounts : t = + match to_yojson t |> Util.member "genesis_ledger" with + | `List accounts -> + let accounts = + List.map accounts ~f:(function + | `Assoc account -> + `Assoc + (List.map account ~f:(fun (key, value) -> + ( key + , if String.equal key "timing" then + Mina_base.Account_timing.of_yojson value + |> Result.ok_or_failwith |> convert + |> timed_opt_to_yojson + else value ) ) ) + | _ -> + failwith "Invalid account json" ) + in + `List accounts + | _ -> + failwith "Invalid genesis ledger accounts" + in + to_string accounts + |> sprintf {| { "accounts": %s } |} + |> from_string |> Runtime_config.Ledger.of_yojson + +let generate_pk () = + let open Signature_lib in + (Keypair.create ()).public_key |> Public_key.compress + |> Public_key.Compressed.to_base58_check + +let topology_of_test_config t : Yojson.Safe.t = + let topology_of_archive { Archive_node.node_name; _ } = + { alias = node_name; pk = generate_pk (); role = Archive_node } + in + let topology_of_block_producer { Block_producer_node.node_name; _ } = + { alias = node_name; pk = generate_pk (); role = Block_producer } + in + let topology_of_seed { Seed_node.node_name; _ } = + { alias = node_name; pk = generate_pk (); role = Seed_node } + in + let topology_of_snark_coordinator { Snark_coordinator_node.node_name; _ } = + { alias = node_name; pk = generate_pk (); role = Snark_coordinator } + in + let topology_of_snark_worker { Snark_worker_node.node_name; _ } = + { alias = node_name; pk = generate_pk (); role = Snark_worker } + in + let snark_coordinator = + match Option.map t.snark_coordinator ~f:topology_of_snark_coordinator with + | None -> + [] + | Some sc -> + [ sc ] + in + let open List in + snark_coordinator + @ map t.archive_nodes ~f:topology_of_archive + @ map t.block_producers ~f:topology_of_block_producer + @ map t.seed_nodes ~f:topology_of_seed + @ map t.snark_workers ~f:topology_of_snark_worker + |> topology_to_yojson + +let test_account ?(pk = generate_pk ()) + ?(timing = Mina_base.Account.Timing.Untimed) account_name balance : + Test_Account.t = + { account_name; balance; timing; pk } + +let topology_info ~alias ~pk ~role : topology_node = { alias; pk; role } + +module Unit_tests = struct + let test_config = + { default with + genesis_ledger = + [ test_account "receiver-key" "9999999" + ; test_account "empty-bp-key" "0" + ; test_account "snark-node-key" "0" + ] + @ List.init 2 ~f:(fun i -> + let i_str = Int.to_string i in + test_account (sprintf "sender-account%s" i_str) "10000" ) + ; block_producers = + [ { node_name = "receiver"; account_name = "receiver-key" } + ; { node_name = "empty_node-1"; account_name = "empty-bp-key" } + ; { node_name = "empty_node-2"; account_name = "empty-bp-key" } + ; { node_name = "empty_node-3"; account_name = "empty-bp-key" } + ; { node_name = "empty_node-4"; account_name = "empty-bp-key" } + ; { node_name = "observer"; account_name = "empty-bp-key" } + ] + ; snark_coordinator = + Some + { node_name = "snark-node" + ; account_name = "snark-node-key" + ; worker_nodes = 4 + } + ; txpool_max_size = 10_000_000 + ; snark_worker_fee = "0.0001" + ; proof_config = + { proof_config_default with + work_delay = Some 1 + ; transaction_capacity = + Some Runtime_config.Proof_keys.Transaction_capacity.small + } + } + + let%test_unit "runtime_config_of_test_config" = + let () = + print_endline "=== Runtime config ===" ; + runtime_config_of_test_config test_config + |> Result.ok_or_failwith |> Runtime_config.Ledger.to_yojson + |> Yojson.Safe.pretty_to_string |> print_endline + in + ignore + ( runtime_config_of_test_config test_config + |> Result.ok_or_failwith |> Runtime_config.Ledger.to_yojson + : Yojson.Safe.t ) + + let%test_unit "topology_of_test_config" = + let () = + print_endline "\n=== Topology ===" ; + topology_of_test_config test_config + |> Yojson.Safe.pretty_to_string |> print_endline + in + ignore (topology_of_test_config test_config : Yojson.Safe.t) +end From 501f54c3948617dd2d12a016a33b432cd18ec918 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 29 Aug 2023 17:47:37 +0000 Subject: [PATCH 42/90] (unit test) Move config.json to test_data dir --- .../test_data/config.json | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 src/lib/integration_test_abstract_engine/test_data/config.json diff --git a/src/lib/integration_test_abstract_engine/test_data/config.json b/src/lib/integration_test_abstract_engine/test_data/config.json new file mode 100644 index 00000000000..06951e3f21b --- /dev/null +++ b/src/lib/integration_test_abstract_engine/test_data/config.json @@ -0,0 +1,95 @@ +{ + "version": 1, + "actions": [ + { + "name": "create_network", + "args": {}, + "command": "minimina network create" + }, + { + "name": "start_network", + "args": { + "network_id": "string" + }, + "command": "minimina network start --network-id {{network_id}}" + }, + { + "name": "delete_network", + "args": { + "network_id": "string" + }, + "command": "minimina network delete --network-id {{network_id}}" + }, + { + "name": "network_status", + "args": { + "network_id": "string" + }, + "command": "minimina network status --network-id {{network_id}}" + }, + { + "name": "start_node", + "args": { + "network_id":"string", + "node_id":"string", + "fresh_state":"bool", + "git_commit":"string" + }, + "command": "minimina node start --network-id {{network_id}} --node-id {{node_id}} {{fresh_state}} --git-commit {{git_commit}}" + }, + { + "name": "stop_node", + "args": { + "network_id":"string", + "node_id":"string" + }, + "command": "minimina node stop --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "dump_archive_data", + "args": { + "network_id":"string", + "node_id":"string" + }, + "command": "minimina node dump-archive-data --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "dump_mina_logs", + "args": { + "network_id":"string", + "node_id":"string" + }, + "command": "minimina node dump-mina-logs --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "dump_precomputed_blocks", + "args": { + "network_id":"string", + "node_id":"string" + }, + "command": "minimina node dump-precomputed-blocks --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "run_replayer", + "args": { + "network_id":"string", + "node_id":"string" + }, + "command": "minimina node run-replayer --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "echo", + "args": { + "msg": "string" + }, + "command": "echo \"{\"msg\":\"{{msg}}\"}\"" + }, + { + "name": "cat", + "args": { + "path": "string" + }, + "command": "cat {{path}}" + } + ] +} From 934007355802263da3c6aaabc946b4006d1c4ede Mon Sep 17 00:00:00 2001 From: Quantifier Date: Fri, 1 Sep 2023 21:58:02 +0000 Subject: [PATCH 43/90] (config) Update config and return spec fields --- integration_tests/mock_config.json | 10 +++++----- integration_tests/return_spec.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integration_tests/mock_config.json b/integration_tests/mock_config.json index 3a3f7ea09cb..62052374fff 100644 --- a/integration_tests/mock_config.json +++ b/integration_tests/mock_config.json @@ -5,9 +5,10 @@ "name": "create_network", "args": { "network_id": "string", - "test_config": "string" + "runtime_config": "string", + "topology": "string" }, - "command": "MOCK_NETWORK network create --network-id {{network_id}} --test-config {{test_config}}" + "command": "MOCK_NETWORK network create --network-id {{network_id}} --runtime-config {{runtime_config}} --topology {{topology}}" }, { "name": "start_network", @@ -28,10 +29,9 @@ "args": { "network_id": "string", "node_id": "string", - "fresh_state": "bool", - "git_commit": "string" + "fresh_state": "bool" }, - "command": "MOCK_NETWORK node start {{fresh_state}} --git-commit {{git_commit}} --network-id {{network_id}} --node-id {{node_id}}" + "command": "MOCK_NETWORK node start {{fresh_state}} --network-id {{network_id}} --node-id {{node_id}}" }, { "name": "stop_node", diff --git a/integration_tests/return_spec.json b/integration_tests/return_spec.json index 9c9df4b4b20..fc111f1efb9 100644 --- a/integration_tests/return_spec.json +++ b/integration_tests/return_spec.json @@ -3,7 +3,7 @@ "return_spec": { "create_network": { "network_id": "string", - "nodes_map": [ + "nodes": [ { "graphql_uri" : "string", "network_id" : "string", From c6d6d3e343bbc4a9e8e9ee1f4818274eda7908a9 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Fri, 1 Sep 2023 22:08:38 +0000 Subject: [PATCH 44/90] (unit test) Move example config file --- .../config.json | 95 ------------------- .../unit_tests.ml | 58 ++++++----- 2 files changed, 28 insertions(+), 125 deletions(-) delete mode 100644 src/lib/integration_test_abstract_engine/config.json diff --git a/src/lib/integration_test_abstract_engine/config.json b/src/lib/integration_test_abstract_engine/config.json deleted file mode 100644 index 06951e3f21b..00000000000 --- a/src/lib/integration_test_abstract_engine/config.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "version": 1, - "actions": [ - { - "name": "create_network", - "args": {}, - "command": "minimina network create" - }, - { - "name": "start_network", - "args": { - "network_id": "string" - }, - "command": "minimina network start --network-id {{network_id}}" - }, - { - "name": "delete_network", - "args": { - "network_id": "string" - }, - "command": "minimina network delete --network-id {{network_id}}" - }, - { - "name": "network_status", - "args": { - "network_id": "string" - }, - "command": "minimina network status --network-id {{network_id}}" - }, - { - "name": "start_node", - "args": { - "network_id":"string", - "node_id":"string", - "fresh_state":"bool", - "git_commit":"string" - }, - "command": "minimina node start --network-id {{network_id}} --node-id {{node_id}} {{fresh_state}} --git-commit {{git_commit}}" - }, - { - "name": "stop_node", - "args": { - "network_id":"string", - "node_id":"string" - }, - "command": "minimina node stop --network-id {{network_id}} --node-id {{node_id}}" - }, - { - "name": "dump_archive_data", - "args": { - "network_id":"string", - "node_id":"string" - }, - "command": "minimina node dump-archive-data --network-id {{network_id}} --node-id {{node_id}}" - }, - { - "name": "dump_mina_logs", - "args": { - "network_id":"string", - "node_id":"string" - }, - "command": "minimina node dump-mina-logs --network-id {{network_id}} --node-id {{node_id}}" - }, - { - "name": "dump_precomputed_blocks", - "args": { - "network_id":"string", - "node_id":"string" - }, - "command": "minimina node dump-precomputed-blocks --network-id {{network_id}} --node-id {{node_id}}" - }, - { - "name": "run_replayer", - "args": { - "network_id":"string", - "node_id":"string" - }, - "command": "minimina node run-replayer --network-id {{network_id}} --node-id {{node_id}}" - }, - { - "name": "echo", - "args": { - "msg": "string" - }, - "command": "echo \"{\"msg\":\"{{msg}}\"}\"" - }, - { - "name": "cat", - "args": { - "path": "string" - }, - "command": "cat {{path}}" - } - ] -} diff --git a/src/lib/integration_test_abstract_engine/unit_tests.ml b/src/lib/integration_test_abstract_engine/unit_tests.ml index 5f9a1d3d9ad..73c486f9b8b 100644 --- a/src/lib/integration_test_abstract_engine/unit_tests.ml +++ b/src/lib/integration_test_abstract_engine/unit_tests.ml @@ -8,7 +8,7 @@ module Test_values = struct let cwd_list = String.split ~on:'/' @@ Sys.getcwd () in let dir_path = List.take cwd_list (List.length cwd_list - 5) - @ [ "src/lib/integration_test_abstract_engine/config.json" ] + @ [ "src/lib/integration_test_abstract_engine/test_data/config.json" ] in String.concat ~sep:"/" dir_path @@ -220,8 +220,9 @@ module Run_command_tests = struct @@ run_command ~suppress_logs:true ~config:config_file ~args:[ ("msg0", `String "hello") ] "echo" - with Missing_arg (arg_name, arg_type) -> - assert (arg_name = "msg" && arg_type = Arg_type.string) + with Missing_arg (action_name, arg_name, arg_type) -> + assert ( + action_name = "echo" && arg_name = "msg" && arg_type = Arg_type.string ) end module Successful = struct @@ -346,38 +347,33 @@ module Parse_output_tests = struct let result = {| { "network_id": "network0", - "nodes": [ - { "node0": { - "graphql_uri": "gql_archive", - "node_type": "Archive_node", - "private_key": null - } + "nodes": { + "node0": { + "graphql_uri": "gql_archive", + "node_type": "Archive_node", + "private_key": null }, - { "node1": { - "graphql_uri": "gql_bp", - "node_type": "Block_producer_node", - "private_key": null - } + "node1": { + "graphql_uri": "gql_bp", + "node_type": "Block_producer_node", + "private_key": null }, - { "node2": { - "graphql_uri": "gql_seed", - "node_type": "Seed_node", - "private_key": "EKEQpDAjj7dP3j7fQy4qBU7Kxns85wwq5xMn4zxdyQm83pEWzQ62" - } + "node2": { + "graphql_uri": "gql_seed", + "node_type": "Seed_node", + "private_key": "EKEQpDAjj7dP3j7fQy4qBU7Kxns85wwq5xMn4zxdyQm83pEWzQ62" }, - { "node3": { - "graphql_uri": "gql_snark", - "node_type": "Snark_worker", - "private_key": null - } + "node3": { + "graphql_uri": "gql_snark", + "node_type": "Snark_worker", + "private_key": null }, - { "node4": { - "graphql_uri": null, - "node_type": "Snark_coordinator", - "private_key": null - } + "node4": { + "graphql_uri": null, + "node_type": "Snark_coordinator", + "private_key": null } - ] + } } |} |> Yojson.Safe.from_string |> of_yojson |> Result.ok_or_failwith @@ -551,3 +547,5 @@ module Parse_output_tests = struct end let () = ignore @@ Async.Scheduler.go () + +let () = print_endline "DONE" From 5f2dbb5aa6058abe00f7449f42cc977c47efd3ca Mon Sep 17 00:00:00 2001 From: Quantifier Date: Fri, 1 Sep 2023 22:11:42 +0000 Subject: [PATCH 45/90] (intg test) Pull local keyapirs for abstract engine --- src/app/test_executive/test_executive.ml | 64 ++++++- .../ci_interaction.ml | 164 +++++++++++------- 2 files changed, 163 insertions(+), 65 deletions(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index cd2c148a3f7..240d8b7046f 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -31,6 +31,7 @@ type inputs = ; archive_image : string option ; debug : bool ; config_path : string option + ; keypairs_path : string ; mock_alias : (string * string) option } @@ -45,7 +46,8 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : [%log fatal] "mina-image argument cannot be an empty string" ; exit 1 ) else if - test_config.num_archive_nodes > 0 && Option.is_none inputs.archive_image + List.length test_config.archive_nodes > 0 + && Option.is_none inputs.archive_image then ( [%log fatal] "This test uses archive nodes. archive-image argument cannot be absent \ @@ -64,6 +66,36 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : "Must provide either --mock command line arg or set MOCK_NETWORK env \ var" ; ignore @@ exit 1 ) ; + let keypairs_path = + if String.(suffix inputs.keypairs_path 1 = "/") then + String.drop_suffix inputs.keypairs_path 1 + else inputs.keypairs_path + in + let keypairs_ls = Stdlib.Sys.readdir keypairs_path in + (* check network-keypairs *) + if + not + ( Array.exists keypairs_ls ~f:(String.equal "network-keypairs") + && (Stdlib.Sys.is_directory @@ keypairs_path ^ "/network-keypairs") ) + then ( + [%log fatal] + "No network-keypairs directory present in %s \n\ + \ Consider cloning the pre-generated keypairs repo: \n\ + \ git clone git@github.com:MinaFoundation/lucy-keypairs.git" + keypairs_path ; + ignore @@ exit 1 ) ; + (* check libp2p-keypairs *) + if + not + ( Array.exists keypairs_ls ~f:(String.equal "libp2p-keypairs") + && (Stdlib.Sys.is_directory @@ keypairs_path ^ "/libp2p-keypairs") ) + then ( + [%log fatal] + "No libp2p-keypairs directory present in %s \n\ + \ Consider cloning the pre-generated keypairs repo: \n\ + \ git clone git@github.com:MinaFoundation/lucy-keypairs.git" + keypairs_path ; + ignore @@ exit 1 ) ; match Inputs.Engine.name with | "abstract" -> if Option.is_none inputs.config_path then ( @@ -449,6 +481,14 @@ let mock_alias_arg = & opt (some string) None & info [ "mock-network"; "mock"; "alias" ] ~env ~docv:"MOCK_NETWORK" ~doc) +let keypair_dir_path_arg = + let doc = "Path to the pre-generated network and libp2p keypair directory." in + let env = Arg.env_var "MINA_KEYPAIRS_PATH" ~doc in + Arg.( + required + & opt (some dir) None + & info [ "keypairs-path" ] ~env ~docv:"MINA_KEYPAIRS_PATH" ~doc) + let mina_image_arg = let doc = "Identifier of the Mina docker image to test." in let env = Arg.env_var "MINA_IMAGE" ~doc in @@ -493,6 +533,18 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = Option.iter path ~f:(fun p -> Engine.Network.config_path := p) ; path in + let set_keypair path = + Engine.Network.keypairs_path := path ; + path + in + let set_mina_image image = + Engine.Network.mina_image := image ; + image + in + let set_archive_image image = + Engine.Network.archive_image := image ; + image + in let set_alias alias = let alias = Option.map alias ~f:(fun a -> ("MOCK_NETWORK", a)) in Engine.Network.alias := alias ; @@ -505,22 +557,26 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = Term.(const wrap_cli_inputs $ Engine.Network_config.Cli_inputs.term) in let inputs_term = - let cons_inputs test_inputs test archive_image debug mina_image config_path - mock_alias = + let cons_inputs test_inputs test debug archive_image mina_image config_path + keypairs_path mock_alias = { test_inputs ; test ; mina_image ; archive_image ; debug ; config_path + ; keypairs_path ; mock_alias } in Term.( const cons_inputs $ test_inputs_with_cli_inputs_arg $ test_arg (module Inputs) - $ archive_image_arg $ debug_arg $ mina_image_arg + $ debug_arg + $ (const set_archive_image $ archive_image_arg) + $ (const set_mina_image $ mina_image_arg) $ (const set_config $ config_path_arg) + $ (const set_keypair $ keypair_dir_path_arg) $ (const set_alias $ mock_alias_arg)) in (Term.(const start $ inputs_term), info) diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/ci_interaction.ml index 9f6edec0f07..286ec354704 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/ci_interaction.ml @@ -8,9 +8,18 @@ open Integration_test_lib (* [alias] is instantiated when command line args are parsed *) let alias = ref None +(* [archive_image] is instantiated when command line args are parsed *) +let archive_image : string option ref = ref None + (* [config_path] is instantiated when command line args are parsed *) let config_path = ref "" +(* [keypairs_path] is instantiated when command line args are parsed *) +let keypairs_path = ref "" + +(* [mina_image] is instantiated when command line args are parsed *) +let mina_image = ref "" + module Network_config = struct module Cli_inputs = Cli_inputs @@ -61,16 +70,54 @@ module Network_config = struct let testnet_log_filter t = t.config.network_id - (* TODO: add config file command call here *) + let pull_keypairs num_keypairs = + let int_list = List.range ~stop:`inclusive 1 10_000 in + let random_nums = + Quickcheck.Generator.(of_list int_list |> list_with_length num_keypairs) + |> Quickcheck.random_value + in + let normalize_path path = + if String.(suffix path 1 = "/") then String.drop_suffix path 1 else path + in + (* network keypairs + private keys *) + let keypairs_path = normalize_path !keypairs_path in + let base_filename n = + sprintf "%s/network-keypairs/sender-account-%d.json" keypairs_path n + in + let read_sk n = In_channel.read_all (base_filename n ^ ".sk") in + let read_keypair n = + let open Yojson.Safe in + let json = from_file (base_filename n) in + let sk = read_sk n |> Private_key.of_base58_check_exn in + { Network_keypair.keypair = Keypair.of_private_key_exn sk + ; keypair_name = Util.member "keypair_name" json |> to_string + ; privkey_password = Util.member "privkey_password" json |> to_string + ; public_key = Util.member "public_key" json |> to_string + ; private_key = Util.member "private_key" json |> to_string + } + in + (* libp2p keypairs *) + let libp2p_base_filename n = + sprintf "%s/libp2p-keypairs/libp2p-keypair-%d.json" keypairs_path n + in + let read_peerid n = + In_channel.read_all (libp2p_base_filename n ^ ".peerid") + in + let read_libp2p n = Yojson.Safe.from_file (libp2p_base_filename n) in + ( List.map random_nums ~f:read_keypair + , List.map random_nums ~f:read_sk + , List.map random_nums ~f:read_libp2p + , List.map random_nums ~f:read_peerid ) + let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) = let { requires_graphql ; genesis_ledger + ; archive_nodes ; block_producers ; snark_coordinator ; snark_worker_fee - ; num_archive_nodes ; log_precomputed_blocks ; proof_config ; Test_config.k @@ -78,6 +125,7 @@ module Network_config = struct ; slots_per_epoch ; slots_per_sub_window ; txpool_max_size + ; _ } = test_config in @@ -88,7 +136,7 @@ module Network_config = struct let user_len = Int.min 5 (String.length user_sanitized) in let user = String.sub user_sanitized ~pos:0 ~len:user_len in let git_commit = Mina_version.commit_id_short in - let network_id = "it-" ^ user ^ "-" ^ git_commit ^ "-" ^ test_name in + let network_id = sprintf "it-%s-%s-%s" user git_commit test_name in (* check to make sure the test writer hasn't accidentally created duplicate names of accounts and keys *) let key_names_list = List.map genesis_ledger ~f:(fun acct -> acct.account_name) @@ -105,29 +153,25 @@ module Network_config = struct failwith "All nodes in testnet must have unique names. Check to make sure you \ are not using the same node_name more than once" ; - - (* GENERATE ACCOUNTS AND KEYPAIRS *) - let keypairs = - List.take - (* the first keypair is the genesis winner and is assumed to be untimed. Therefore dropping it, and not assigning it to any block producer *) - (List.drop - (Array.to_list (Lazy.force Key_gen.Sample_keypairs.keypairs)) - 1 ) - (List.length genesis_ledger) + (* ACCOUNTS AND KEYPAIRS *) + let before = Time.now () in + let num_keypairs = List.length genesis_ledger in + let network_keypairs, private_keys, _libp2p_keypairs, _libp2p_peerids = + pull_keypairs (List.length genesis_ledger) in + [%log trace] "Pulled %d keypairs from %s in %s" num_keypairs !keypairs_path + Time.(abs_diff before @@ now () |> Span.to_string) ; let labeled_accounts : - ( Runtime_config.Accounts.single - * (Public_key.Compressed.t * Private_key.t) ) + (Runtime_config.Accounts.single * (Network_keypair.t * Private_key.t)) String.Map.t = String.Map.empty in - let rec add_accounts mp zip = - match zip with + let rec add_accounts acc = function | [] -> - mp + acc | hd :: tl -> - let ( { Test_config.Test_Account.balance; account_name; timing } - , (pk, sk) ) = + let ( { Test_config.Test_Account.balance; account_name; timing; _ } + , (network_keypair, sk) ) = hd in let timing = @@ -145,6 +189,8 @@ module Network_config = struct } in let default = Runtime_config.Accounts.Single.default in + let sk = Private_key.of_base58_check_exn sk in + let pk = Public_key.(of_private_key_exn sk |> compress) in let acct = { default with pk = Public_key.Compressed.to_string pk @@ -157,10 +203,18 @@ module Network_config = struct } in add_accounts - (String.Map.add_exn mp ~key:account_name ~data:(acct, (pk, sk))) + (String.Map.add_exn acc ~key:account_name + ~data:(acct, (network_keypair, sk)) ) tl in let genesis_ledger_accounts = + let network_keypairs = + List.mapi network_keypairs ~f:(fun n kp -> + { kp with + keypair_name = (List.nth_exn genesis_ledger n).account_name + } ) + in + let keypairs = List.zip_exn network_keypairs private_keys in add_accounts labeled_accounts (List.zip_exn genesis_ledger keypairs) in (* DAEMON CONFIG *) @@ -187,15 +241,14 @@ module Network_config = struct ; slots_per_epoch = Some slots_per_epoch ; slots_per_sub_window = Some slots_per_sub_window ; genesis_state_timestamp = - Some Core.Time.(to_string_abs ~zone:Zone.utc (now ())) + Some Core.Time.(to_string_abs ~zone:Zone.utc @@ now ()) } ; proof = Some proof_config (* TODO: prebake ledger and only set hash *) ; ledger = Some { base = Accounts - ( String.Map.data genesis_ledger_accounts - |> List.map ~f:(fun tup -> fst tup) ) + (String.Map.data genesis_ledger_accounts |> List.map ~f:fst) ; add_genesis_winner = None ; num_accounts = None ; balances = [] @@ -214,18 +267,12 @@ module Network_config = struct { constraints = constraint_constants; genesis = genesis_constants } in (* BLOCK PRODUCER CONFIG *) - let mk_net_keypair keypair_name (pk, sk) = - let keypair = - { Keypair.public_key = Public_key.decompress_exn pk; private_key = sk } - in - Network_keypair.create_network_keypair ~keypair_name ~keypair - in let block_producer_config name keypair = { name; keypair; libp2p_secret = "" } in let block_producer_configs = - List.map block_producers ~f:(fun node -> - let _, key_tup = + List.mapi block_producers ~f:(fun n node -> + let _ = match String.Map.find genesis_ledger_accounts node.account_name with | Some acct -> acct @@ -240,14 +287,13 @@ module Network_config = struct failwith failstring in block_producer_config node.node_name - (mk_net_keypair node.account_name key_tup) ) + @@ List.nth_exn network_keypairs n ) in let mina_archive_schema = "create_schema.sql" in let long_commit_id = - if String.is_substring Mina_version.commit_id ~substring:"[DIRTY]" then - String.sub Mina_version.commit_id ~pos:7 - ~len:(String.length Mina_version.commit_id - 7) - else Mina_version.commit_id + let open String in + let id = Mina_version.commit_id in + if prefix id 7 = "[DIRTY]" then suffix id (length id - 7) else id in let mina_archive_base_url = "https://raw.githubusercontent.com/MinaProtocol/mina/" ^ long_commit_id @@ -258,17 +304,13 @@ module Network_config = struct ; mina_archive_base_url ^ "zkapp_tables.sql" ] in - let before = Time.now () in let genesis_keypairs = String.Map.of_alist_exn (List.map (String.Map.to_alist genesis_ledger_accounts) ~f:(fun element -> - let kp_name, (_, (pk, sk)) = element in - (kp_name, mk_net_keypair kp_name (pk, sk)) ) ) + let kp_name, (_, (keypair, _)) = element in + (kp_name, keypair) ) ) in - [%log trace] "Genesis keypairs (%d): %s" - (String.Map.length genesis_keypairs) - Time.(abs_diff before @@ now () |> Span.to_string) ; let snark_coordinator_config = match snark_coordinator with | None -> @@ -312,7 +354,7 @@ module Network_config = struct ; runtime_config = Runtime_config.to_yojson runtime_config ; block_producer_configs ; log_precomputed_blocks - ; archive_node_count = num_archive_nodes + ; archive_node_count = List.length archive_nodes ; mina_archive_schema ; mina_archive_schema_aux_files ; snark_coordinator_config @@ -459,15 +501,14 @@ module Network_deployed = struct raise @@ Invalid_argument (Yojson.Safe.to_string uri) let of_yojson = - let f network_id accum : Yojson.Safe.t -> t = function - | `Assoc - [ ( node_id - , `Assoc - [ ("graphql_uri", graphql_uri) - ; ("node_type", `String nt) - ; ("private_key", private_key) - ] ) - ] -> + let node_info_helper network_id accum : string * Yojson.Safe.t -> t = + function + | ( node_id + , `Assoc + [ ("graphql_uri", graphql_uri) + ; ("node_type", `String nt) + ; ("private_key", private_key) + ] ) -> Core.String.Map.set accum ~key:node_id ~data: { node_id @@ -476,15 +517,15 @@ module Network_deployed = struct ; network_keypair = network_keypair_of_yojson node_id private_key ; graphql_uri = graphql_uri_of_yojson graphql_uri } - | t -> - raise @@ Invalid_entry (Yojson.Safe.to_string t) + | node_id, t -> + raise + @@ Invalid_entry (sprintf "%s: %s" node_id @@ Yojson.Safe.to_string t) in function - | `Assoc [ ("network_id", `String network_id); ("nodes", `List nodes) ] -> - Ok (List.fold nodes ~init:Core.String.Map.empty ~f:(f network_id)) - | `Assoc [ ("network_id", `String _network_id) ] -> - (* TODO: remove *) - Ok Core.String.Map.empty + | `Assoc [ ("network_id", `String network_id); ("nodes", `Assoc nodes) ] -> + Ok + (List.fold nodes ~init:Core.String.Map.empty + ~f:(node_info_helper network_id) ) | t -> print_endline @@ Yojson.Safe.pretty_to_string t ; raise @@ Invalid_output (Yojson.Safe.to_string t) @@ -613,7 +654,7 @@ module Config_file = struct exception Invalid_args_num of string * string list * string list - exception Missing_arg of string * string + exception Missing_arg of string * string * string exception Invalid_arg_type of string * Yojson.Safe.t * string @@ -681,7 +722,8 @@ module Config_file = struct try List.find_map_exn args ~f:(fun (name, value) -> Option.some_if (String.equal name arg_name) value ) - with Not_found_s _ -> raise @@ Missing_arg (arg_name, arg_type) + with Not_found_s _ -> + raise @@ Missing_arg (action_name action, arg_name, arg_type) in if validate_arg_type arg_type arg_value then aux rest else raise @@ Invalid_arg_type (arg_name, arg_value, arg_type) From b16aef465e5b95e124b600f2f8b7aaabc35c8f27 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Fri, 1 Sep 2023 22:13:15 +0000 Subject: [PATCH 46/90] (intg test) Enrich Test_config with topology info for abstract engine --- .../abstract_network.ml | 10 +- .../mina_automation.ml | 48 +++-- .../kubernetes_network.ml | 9 +- .../mina_automation.ml | 15 +- src/lib/integration_test_lib/intf.ml | 12 +- src/lib/integration_test_lib/test_config.ml | 202 ++++++++++++++---- src/lib/runtime_config/runtime_config.ml | 2 +- 7 files changed, 231 insertions(+), 67 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index 505fbd4386f..f33c82883e5 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -9,8 +9,14 @@ open Ci_interaction let config_path = config_path +let keypairs_path = keypairs_path + +let mina_image = mina_image + let alias = alias +let archive_image = archive_image + module Node = struct type t = { node_id : Node_id.t @@ -25,12 +31,10 @@ module Node = struct let network_keypair { network_keypair; _ } = network_keypair - let start ?(git_commit = Mina_version.commit_id_short) ~fresh_state t : - unit Malleable_error.t = + let start ~fresh_state t : unit Malleable_error.t = try let args = [ ("fresh_state", `Bool fresh_state) - ; ("git_commit", `String git_commit) ; ("network_id", `String t.network_id) ; ("node_id", `String t.node_id) ] diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index 9e31a7e17fe..35b82b19324 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -14,7 +14,8 @@ module Network_manager = struct ; constants : Test_config.constants ; mutable deployed : bool ; genesis_keypairs : Network_keypair.t Core.String.Map.t - ; test_config_path : string + ; runtime_config_path : string + ; topology_path : string } let run_cmd t prog args = Util.run_cmd t.testnet_dir prog args @@ -46,11 +47,21 @@ module Network_manager = struct in let open Deferred.Let_syntax in [%log info] "Making testnet dir %s" testnet_dir ; - let%bind exists = Sys.file_exists_exn testnet_dir in - ignore @@ if exists then Unix.remove testnet_dir else return () ; + let rec rmrf path = + let open Stdlib in + match Sys.is_directory path with + | true -> + Sys.readdir path + |> Array.iter (fun name -> rmrf (Filename.concat path name)) ; + Core.Unix.rmdir path + | false -> + Sys.remove path + in + if Stdlib.Sys.file_exists testnet_dir then rmrf testnet_dir ; let () = mkdir_p testnet_dir in let network_config_filename = testnet_dir ^/ "network_config.json" in - let test_config_filename = testnet_dir ^/ "test_config.json" in + let runtime_config_filename = testnet_dir ^/ "runtime_config.json" in + let topology_filename = testnet_dir ^/ "topology.json" in let t = { logger ; network_id = network_config.config.network_id @@ -60,28 +71,32 @@ module Network_manager = struct ; constants = network_config.constants ; deployed = false ; genesis_keypairs = network_config.genesis_keypairs - ; test_config_path = test_config_filename + ; runtime_config_path = runtime_config_filename + ; topology_path = topology_filename } in - [%log info] "Writing network configuration into %s" network_config_filename ; + [%log info] "Writing network configuration to %s" network_config_filename ; Out_channel.with_file ~fail_if_exists:true network_config_filename ~f:(fun ch -> Network_config.to_yojson network_config |> Yojson.Safe.to_channel ch ) ; - [%log info] "Writing test configuration into %s" test_config_filename ; - Out_channel.with_file ~fail_if_exists:true test_config_filename + [%log info] "Writing runtime configuration to %s" runtime_config_filename ; + Out_channel.with_file ~fail_if_exists:true runtime_config_filename ~f:(fun ch -> - Test_config.to_yojson test_config |> Yojson.Safe.to_channel ch ) ; - [%log info] - "Writing out the genesis keys (in case you want to use them manually) to \ - testnet dir %s" - testnet_dir ; - let kps_base_path = String.concat [ testnet_dir; "/genesis_keys" ] in + Test_config.runtime_config_of_test_config test_config + |> Result.ok_or_failwith |> Runtime_config.Ledger.to_yojson + |> Yojson.Safe.to_channel ch ) ; + [%log info] "Writing topology to %s" topology_filename ; + Out_channel.with_file ~fail_if_exists:true topology_filename ~f:(fun ch -> + Test_config.topology_of_test_config test_config + |> Yojson.Safe.to_channel ch ) ; + [%log info] "Writing out the genesis keys to testnet dir %s" testnet_dir ; + let kps_base_path = testnet_dir ^ "/genesis_keys" in let%bind () = Unix.mkdir kps_base_path in let%bind () = Core.String.Map.iter network_config.genesis_keypairs ~f:(fun kp -> Network_keypair.to_yojson kp |> Yojson.Safe.to_file - (String.concat [ kps_base_path; "/"; kp.keypair_name; ".json" ]) ) + (sprintf "%s/%s.json" kps_base_path kp.keypair_name) ) |> Deferred.return in Malleable_error.return t @@ -97,7 +112,8 @@ module Network_manager = struct ~config:!Abstract_network.config_path ~args: [ ("network_id", `String t.network_id) - ; ("test_config", `String t.test_config_path) + ; ("runtime_config", `String t.runtime_config_path) + ; ("topology", `String t.topology_path) ] "create_network" with diff --git a/src/lib/integration_test_cloud_engine/kubernetes_network.ml b/src/lib/integration_test_cloud_engine/kubernetes_network.ml index dbbbddb413c..c5de2826d5f 100644 --- a/src/lib/integration_test_cloud_engine/kubernetes_network.ml +++ b/src/lib/integration_test_cloud_engine/kubernetes_network.ml @@ -7,8 +7,14 @@ open Integration_test_lib let alias : (string * string) option ref = ref None +let archive_image : string option ref = ref None + let config_path = ref "N/A" +let keypairs_path = ref "" + +let mina_image = ref "" + let id _ = "cloud" let mina_archive_container_id = "archive" @@ -111,8 +117,7 @@ module Node = struct Integration_test_lib.Util.run_cmd_or_error cwd "kubectl" (base_kube_args config @ [ "cp"; "-c"; container_id; tmp_file; dest_file ]) - let[@warning "-27"] start ?(git_commit = "") ~fresh_state node : - unit Malleable_error.t = + let start ~fresh_state node : unit Malleable_error.t = let open Malleable_error.Let_syntax in node.should_be_running <- true ; let%bind () = diff --git a/src/lib/integration_test_cloud_engine/mina_automation.ml b/src/lib/integration_test_cloud_engine/mina_automation.ml index 3fcf493d80b..72388dea5ea 100644 --- a/src/lib/integration_test_cloud_engine/mina_automation.ml +++ b/src/lib/integration_test_cloud_engine/mina_automation.ml @@ -113,10 +113,12 @@ module Network_config = struct let { requires_graphql ; genesis_ledger ; epoch_data + ; archive_nodes ; block_producers + ; seed_nodes ; snark_coordinator + ; snark_workers ; snark_worker_fee - ; num_archive_nodes ; log_precomputed_blocks (* ; num_plain_nodes *) ; proof_config ; Test_config.k @@ -127,6 +129,15 @@ module Network_config = struct } = test_config in + List.iter archive_nodes ~f:(fun x -> + Test_config.Archive_node.to_yojson x + |> Yojson.Safe.pretty_to_string |> print_endline ) ; + List.iter seed_nodes ~f:(fun x -> + Test_config.Seed_node.to_yojson x + |> Yojson.Safe.pretty_to_string |> print_endline ) ; + List.iter snark_workers ~f:(fun x -> + Test_config.Snark_worker_node.to_yojson x + |> Yojson.Safe.pretty_to_string |> print_endline ) ; let user_from_env = Option.value (Unix.getenv "USER") ~default:"auto" in let user_sanitized = Str.global_replace (Str.regexp "\\W|_-") "" user_from_env @@ -442,7 +453,7 @@ module Network_config = struct ; runtime_config = Runtime_config.to_yojson runtime_config ; block_producer_configs ; log_precomputed_blocks - ; archive_node_count = num_archive_nodes + ; archive_node_count = List.length archive_nodes ; mina_archive_schema ; mina_archive_schema_aux_files ; snark_coordinator_config diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index 37caf38e947..a701976cc01 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -51,8 +51,7 @@ module Engine = struct val network_keypair : t -> Network_keypair.t option - val start : - ?git_commit:string -> fresh_state:bool -> t -> unit Malleable_error.t + val start : fresh_state:bool -> t -> unit Malleable_error.t val stop : t -> unit Malleable_error.t @@ -108,8 +107,17 @@ module Engine = struct (*** [alias] is instantiated when command line args are parsed *) val alias : (string * string) option ref + (*** [archive_image] is instantiated when command line args are parsed *) + val archive_image : string option ref + (*** [config_path] is instantiated when command line args are parsed *) val config_path : string ref + + (*** [keypairs_path] is instantiated when command line args are parsed *) + val keypairs_path : string ref + + (*** [mina_image] is instantiated when command line args are parsed *) + val mina_image : string ref end module type Network_manager_intf = sig diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 6962861a031..3b22e25cc76 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -73,24 +73,45 @@ module Epoch_data = struct end module Archive_node = struct - type t = { node_name : string; account_name : string } [@@deriving to_yojson] + type t = { node_name : string; account_name : string; docker_image : string } + [@@deriving to_yojson] end module Block_producer_node = struct - type t = { node_name : string; account_name : string } [@@deriving to_yojson] + type t = + { node_name : string + ; account_name : string + ; libp2p_keypair : Yojson.Safe.t + ; libp2p_peerid : string + ; docker_image : string + } + [@@deriving to_yojson] end module Seed_node = struct - type t = { node_name : string; account_name : string } [@@deriving to_yojson] + type t = + { node_name : string + ; account_name : string + ; libp2p_keypair : Yojson.Safe.t + ; libp2p_peerid : string + ; docker_image : string + } + [@@deriving to_yojson] end module Snark_coordinator_node = struct - type t = { node_name : string; account_name : string; worker_nodes : int } + type t = + { node_name : string + ; account_name : string + ; docker_image : string + ; worker_nodes : int + } [@@deriving to_yojson] end module Snark_worker_node = struct - type t = { node_name : string; account_name : string } [@@deriving to_yojson] + type t = { node_name : string; account_name : string; docker_image : string } + [@@deriving to_yojson] end type constants = @@ -140,10 +161,23 @@ module Node_role = struct `String "Snark_worker" end -type topology = topology_node list +module Topology = struct + type node_info = + { pk : string + ; sk : string + ; role : Node_role.t + ; docker_image : string + ; libp2p_keypair : Yojson.Safe.t + ; libp2p_peerid : Yojson.Safe.t + } + [@@deriving to_yojson] -and topology_node = { alias : string; pk : string; role : Node_role.t } -[@@deriving to_yojson] + type t = (string * node_info) list + + let to_yojson nodes : Yojson.Safe.t = + let alist = List.map nodes ~f:(fun (a, b) -> (a, node_info_to_yojson b)) in + `Assoc alist +end let proof_config_default : Runtime_config.Proof_keys.t = { level = Some Full @@ -279,26 +313,83 @@ let runtime_config_of_test_config t = |> sprintf {| { "accounts": %s } |} |> from_string |> Runtime_config.Ledger.of_yojson -let generate_pk () = +let generate_pk_sk () = let open Signature_lib in - (Keypair.create ()).public_key |> Public_key.compress - |> Public_key.Compressed.to_base58_check + let kp = Keypair.create () in + let sk = kp.private_key |> Private_key.to_base58_check in + let pk = + kp.public_key |> Public_key.compress + |> Public_key.Compressed.to_base58_check + in + (pk, sk) let topology_of_test_config t : Yojson.Safe.t = - let topology_of_archive { Archive_node.node_name; _ } = - { alias = node_name; pk = generate_pk (); role = Archive_node } + let topology_of_archive { Archive_node.node_name; docker_image; _ } : + string * Topology.node_info = + let pk, sk = generate_pk_sk () in + ( node_name + , { pk + ; sk + ; role = Archive_node + ; docker_image + ; libp2p_keypair = `Null + ; libp2p_peerid = `Null + } ) in - let topology_of_block_producer { Block_producer_node.node_name; _ } = - { alias = node_name; pk = generate_pk (); role = Block_producer } + let topology_of_block_producer + { Block_producer_node.node_name + ; docker_image + ; libp2p_keypair + ; libp2p_peerid + ; _ + } : string * Topology.node_info = + let pk, sk = generate_pk_sk () in + ( node_name + , { pk + ; sk + ; role = Block_producer + ; docker_image + ; libp2p_keypair + ; libp2p_peerid = `String libp2p_peerid + } ) in - let topology_of_seed { Seed_node.node_name; _ } = - { alias = node_name; pk = generate_pk (); role = Seed_node } + let topology_of_seed + { Seed_node.node_name; docker_image; libp2p_keypair; libp2p_peerid; _ } : + string * Topology.node_info = + let pk, sk = generate_pk_sk () in + ( node_name + , { pk + ; sk + ; role = Seed_node + ; docker_image + ; libp2p_keypair + ; libp2p_peerid = `String libp2p_peerid + } ) in - let topology_of_snark_coordinator { Snark_coordinator_node.node_name; _ } = - { alias = node_name; pk = generate_pk (); role = Snark_coordinator } + let topology_of_snark_coordinator + { Snark_coordinator_node.node_name; docker_image; _ } : + string * Topology.node_info = + let pk, sk = generate_pk_sk () in + ( node_name + , { pk + ; sk + ; role = Snark_coordinator + ; docker_image + ; libp2p_keypair = `Null + ; libp2p_peerid = `Null + } ) in - let topology_of_snark_worker { Snark_worker_node.node_name; _ } = - { alias = node_name; pk = generate_pk (); role = Snark_worker } + let topology_of_snark_worker { Snark_worker_node.node_name; docker_image; _ } + : string * Topology.node_info = + let pk, sk = generate_pk_sk () in + ( node_name + , { pk + ; sk + ; role = Snark_worker + ; docker_image + ; libp2p_keypair = `Null + ; libp2p_peerid = `Null + } ) in let snark_coordinator = match Option.map t.snark_coordinator ~f:topology_of_snark_coordinator with @@ -313,14 +404,21 @@ let topology_of_test_config t : Yojson.Safe.t = @ map t.block_producers ~f:topology_of_block_producer @ map t.seed_nodes ~f:topology_of_seed @ map t.snark_workers ~f:topology_of_snark_worker - |> topology_to_yojson + |> Topology.to_yojson -let test_account ?(pk = generate_pk ()) +let test_account ?(pk = generate_pk_sk () |> fst) ?(timing = Mina_base.Account.Timing.Untimed) account_name balance : Test_Account.t = { account_name; balance; timing; pk } -let topology_info ~alias ~pk ~role : topology_node = { alias; pk; role } +let bp ?(libp2p_keypair = `Null) ?(libp2p_peerid = "") node_name + ?(account_name = node_name ^ "-key") docker_image = + { Block_producer_node.node_name + ; account_name + ; libp2p_keypair + ; libp2p_peerid + ; docker_image + } module Unit_tests = struct let test_config = @@ -332,19 +430,41 @@ module Unit_tests = struct ] @ List.init 2 ~f:(fun i -> let i_str = Int.to_string i in - test_account (sprintf "sender-account%s" i_str) "10000" ) + test_account (sprintf "sender-account-%s" i_str) "10000" ) ; block_producers = - [ { node_name = "receiver"; account_name = "receiver-key" } - ; { node_name = "empty_node-1"; account_name = "empty-bp-key" } - ; { node_name = "empty_node-2"; account_name = "empty-bp-key" } - ; { node_name = "empty_node-3"; account_name = "empty-bp-key" } - ; { node_name = "empty_node-4"; account_name = "empty-bp-key" } - ; { node_name = "observer"; account_name = "empty-bp-key" } + [ { node_name = "receiver" + ; account_name = "receiver-key" + ; docker_image = "bp-image-0" + ; libp2p_keypair = + Yojson.Safe.from_string + {json| + { + "box_primitive": "xsalsa20poly1305", + "pw_primitive": "argon2i", + "nonce": "8tXZ3yVev59WGBFDhVkXYsqAkAy1oa4a9GkAgke", + "pwsalt": "AbLiWBzfVzdPK2YYRhhUxvQv7W8C", + "pwdiff": [ + 134217728, + 6 + ], + "ciphertext": "8uu1u388h2YJDfQGdAPurUrTgWGiUuRfz2t6TrRPEaqQoPVMHBvZFSC7BMZwCqoWk1PNAQVEPuBLQAhcBtdktmrjBnjhr2P38UiDzSa3gX7uawToQBV1WJh2gTNwPBYh2YRnRZNUY3ThaqqJmx7t7rrApYSqbSoAsLjydGSRYhxdrPtHeokmtcbtzFJ65Q5SWL6vAdhJqCqMcGXUhLDMpCsBwreqkFxn3QmQ9UinG1HTKGNV2FyrXDKUxseboBB3EEmq1Xvmt7ZaPropVcWeL9doWwYVW3xjiMXwR" + } + |json} + ; libp2p_peerid = + "12D3KooWAZBFbTQWmegLFt8YSExEMTwJEiufRf4skYdZtaGxEXXo" + } + ; { node_name = "empty_node" + ; account_name = "empty-bp-key" + ; docker_image = "bp-image-1" + ; libp2p_keypair = `Null + ; libp2p_peerid = "" + } ] ; snark_coordinator = Some { node_name = "snark-node" ; account_name = "snark-node-key" + ; docker_image = "snark-coordinator-image" ; worker_nodes = 4 } ; txpool_max_size = 10_000_000 @@ -357,23 +477,23 @@ module Unit_tests = struct } } + open Yojson.Safe + let%test_unit "runtime_config_of_test_config" = - let () = - print_endline "=== Runtime config ===" ; + let config_json = runtime_config_of_test_config test_config |> Result.ok_or_failwith |> Runtime_config.Ledger.to_yojson - |> Yojson.Safe.pretty_to_string |> print_endline in - ignore - ( runtime_config_of_test_config test_config - |> Result.ok_or_failwith |> Runtime_config.Ledger.to_yojson - : Yojson.Safe.t ) + let () = + print_endline "=== Runtime config ===" ; + pretty_to_string config_json |> print_endline + in + ignore (config_json : t) let%test_unit "topology_of_test_config" = let () = print_endline "\n=== Topology ===" ; - topology_of_test_config test_config - |> Yojson.Safe.pretty_to_string |> print_endline + topology_of_test_config test_config |> pretty_to_string |> print_endline in - ignore (topology_of_test_config test_config : Yojson.Safe.t) + ignore (topology_of_test_config test_config : t) end diff --git a/src/lib/runtime_config/runtime_config.ml b/src/lib/runtime_config/runtime_config.ml index 74af6888f43..cd9f045bb7e 100644 --- a/src/lib/runtime_config/runtime_config.ml +++ b/src/lib/runtime_config/runtime_config.ml @@ -504,7 +504,7 @@ module Accounts = struct ; vesting_period : Mina_numbers.Global_slot_span.Stable.Latest.t ; vesting_increment : Currency.Amount.Stable.Latest.t } - [@@deriving bin_io_unversioned, sexp] + [@@deriving bin_io_unversioned, sexp, to_yojson] end module Permissions = Json_layout.Accounts.Single.Permissions From 832667fe6bd670e6266f6bc463f81dd0e37fc8b5 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Fri, 1 Sep 2023 22:14:39 +0000 Subject: [PATCH 47/90] (intg test) Enrich tests with topology info --- .../block_production_priority.ml | 28 +++++------- src/app/test_executive/block_reward_test.ml | 5 +-- .../test_executive/chain_reliability_test.ml | 12 +++--- src/app/test_executive/gossip_consistency.ml | 8 +--- src/app/test_executive/medium_bootstrap.ml | 12 +++--- src/app/test_executive/mock.ml | 30 ++++++------- src/app/test_executive/payments_test.ml | 43 ++++++++----------- .../test_executive/peers_reliability_test.ml | 12 +++--- src/app/test_executive/snarkyjs.ml | 6 +-- .../test_executive/verification_key_update.ml | 19 +++----- src/app/test_executive/zkapps.ml | 23 ++++------ src/app/test_executive/zkapps_nonce_test.ml | 20 ++++----- src/app/test_executive/zkapps_timing.ml | 23 +++------- 13 files changed, 95 insertions(+), 146 deletions(-) diff --git a/src/app/test_executive/block_production_priority.ml b/src/app/test_executive/block_production_priority.ml index 47353bc09ad..fb50ef450e4 100644 --- a/src/app/test_executive/block_production_priority.ml +++ b/src/app/test_executive/block_production_priority.ml @@ -16,32 +16,26 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Test_config in { default with genesis_ledger = - [ { Test_Account.account_name = "receiver-key" - ; balance = "9999999" - ; timing = Untimed - } - ; { account_name = "empty-bp-key"; balance = "0"; timing = Untimed } - ; { account_name = "snark-node-key"; balance = "0"; timing = Untimed } + [ test_account "receiver-key" "9999999" + ; test_account "empty-bp-key" "0" + ; test_account "snark-node-key" "0" ] @ List.init num_extra_keys ~f:(fun i -> let i_str = Int.to_string i in - { Test_Account.account_name = - String.concat [ "sender-account"; i_str ] - ; balance = "10000" - ; timing = Untimed - } ) + test_account ("sender-account-" ^ i_str) "10000" ) ; block_producers = - [ { node_name = "receiver"; account_name = "receiver-key" } - ; { node_name = "empty_node-1"; account_name = "empty-bp-key" } - ; { node_name = "empty_node-2"; account_name = "empty-bp-key" } - ; { node_name = "empty_node-3"; account_name = "empty-bp-key" } - ; { node_name = "empty_node-4"; account_name = "empty-bp-key" } - ; { node_name = "observer"; account_name = "empty-bp-key" } + [ bp "receiver" !Network.mina_image + ; bp "empty_node-1" ~account_name:"empty-bp-key" !Network.mina_image + ; bp "empty_node-2" ~account_name:"empty-bp-key" !Network.mina_image + ; bp "empty_node-3" ~account_name:"empty-bp-key" !Network.mina_image + ; bp "empty_node-4" ~account_name:"empty-bp-key" !Network.mina_image + ; bp "observer" ~account_name:"empty-bp-key" !Network.mina_image ] ; snark_coordinator = Some { node_name = "snark-node" ; account_name = "snark-node-key" + ; docker_image = !Network.mina_image ; worker_nodes = 4 } ; txpool_max_size = 10_000_000 diff --git a/src/app/test_executive/block_reward_test.ml b/src/app/test_executive/block_reward_test.ml index 63b11d22fc7..cf5275e5d59 100644 --- a/src/app/test_executive/block_reward_test.ml +++ b/src/app/test_executive/block_reward_test.ml @@ -14,9 +14,8 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in { default with - genesis_ledger = - [ { account_name = "node-key"; balance = "1000"; timing = Untimed } ] - ; block_producers = [ { node_name = "node"; account_name = "node-key" } ] + genesis_ledger = [ test_account "node-key" "1000" ] + ; block_producers = [ bp "node" !Network.mina_image ] } let run network t = diff --git a/src/app/test_executive/chain_reliability_test.ml b/src/app/test_executive/chain_reliability_test.ml index 920a773b17d..cbb63949554 100644 --- a/src/app/test_executive/chain_reliability_test.ml +++ b/src/app/test_executive/chain_reliability_test.ml @@ -14,14 +14,14 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Test_config in { default with genesis_ledger = - [ { account_name = "node-a-key"; balance = "1000"; timing = Untimed } - ; { account_name = "node-b-key"; balance = "1000"; timing = Untimed } - ; { account_name = "node-c-key"; balance = "0"; timing = Untimed } + [ test_account "node-a-key" "1000" + ; test_account "node-b-key" "1000" + ; test_account "node-c-key" "0" ] ; block_producers = - [ { node_name = "node-a"; account_name = "node-a-key" } - ; { node_name = "node-b"; account_name = "node-b-key" } - ; { node_name = "node-c"; account_name = "node-c-key" } + [ bp "node-a" !Network.mina_image + ; bp "node-b" !Network.mina_image + ; bp "node-c" !Network.mina_image ] } diff --git a/src/app/test_executive/gossip_consistency.ml b/src/app/test_executive/gossip_consistency.ml index 4229ca320be..9272ccb068f 100644 --- a/src/app/test_executive/gossip_consistency.ml +++ b/src/app/test_executive/gossip_consistency.ml @@ -14,13 +14,9 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Test_config in { default with genesis_ledger = - [ { account_name = "node-a-key"; balance = "1000"; timing = Untimed } - ; { account_name = "node-b-key"; balance = "1000"; timing = Untimed } - ] + [ test_account "node-a-key" "1000"; test_account "node-b-key" "1000" ] ; block_producers = - [ { node_name = "node-a"; account_name = "node-a-key" } - ; { node_name = "node-b"; account_name = "node-b-key" } - ] + [ bp "node-a" !Network.mina_image; bp "node-b" !Network.mina_image ] } let run network t = diff --git a/src/app/test_executive/medium_bootstrap.ml b/src/app/test_executive/medium_bootstrap.ml index b13c0728fb8..9b8925c4932 100644 --- a/src/app/test_executive/medium_bootstrap.ml +++ b/src/app/test_executive/medium_bootstrap.ml @@ -16,14 +16,14 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct { default with k = 2 ; genesis_ledger = - [ { account_name = "node-a-key"; balance = "1000"; timing = Untimed } - ; { account_name = "node-b-key"; balance = "1000"; timing = Untimed } - ; { account_name = "node-c-key"; balance = "0"; timing = Untimed } + [ test_account "node-a-key" "1000" + ; test_account "node-b-key" "1000" + ; test_account "node-c-key" "0" ] ; block_producers = - [ { node_name = "node-a"; account_name = "node-a-key" } - ; { node_name = "node-b"; account_name = "node-b-key" } - ; { node_name = "node-c"; account_name = "node-c-key" } + [ bp "node-a" !Network.mina_image + ; bp "node-b" !Network.mina_image + ; bp "node-c" !Network.mina_image ] } diff --git a/src/app/test_executive/mock.ml b/src/app/test_executive/mock.ml index 651d7205c8a..54125bece3b 100644 --- a/src/app/test_executive/mock.ml +++ b/src/app/test_executive/mock.ml @@ -2,6 +2,8 @@ open Core open Integration_test_lib module Make (Inputs : Intf.Test.Inputs_intf) = struct + open Inputs.Engine + open Test_common.Make (Inputs) let num_extra_keys = 1000 @@ -12,32 +14,26 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Test_config in { default with genesis_ledger = - [ { Test_Account.account_name = "receiver-key" - ; balance = "9999999" - ; timing = Untimed - } - ; { account_name = "empty-bp-key"; balance = "0"; timing = Untimed } - ; { account_name = "snark-node-key"; balance = "0"; timing = Untimed } + [ test_account "receiver-key" "9999999" + ; test_account "empty-bp-key" "0" + ; test_account "snark-node-key" "0" ] @ List.init num_extra_keys ~f:(fun i -> let i_str = Int.to_string i in - { Test_Account.account_name = - String.concat [ "sender-account"; i_str ] - ; balance = "10000" - ; timing = Untimed - } ) + test_account ("sender-account-" ^ i_str) "10000" ) ; block_producers = - [ { node_name = "receiver"; account_name = "receiver-key" } - ; { node_name = "empty_node-1"; account_name = "empty-bp-key" } - ; { node_name = "empty_node-2"; account_name = "empty-bp-key" } - ; { node_name = "empty_node-3"; account_name = "empty-bp-key" } - ; { node_name = "empty_node-4"; account_name = "empty-bp-key" } - ; { node_name = "observer"; account_name = "empty-bp-key" } + [ bp "receiver" !Network.mina_image + ; bp "empty_node-1" ~account_name:"empty-bp-key" !Network.mina_image + ; bp "empty_node-2" ~account_name:"empty-bp-key" !Network.mina_image + ; bp "empty_node-3" ~account_name:"empty-bp-key" !Network.mina_image + ; bp "empty_node-4" ~account_name:"empty-bp-key" !Network.mina_image + ; bp "observer" ~account_name:"empty-bp-key" !Network.mina_image ] ; snark_coordinator = Some { node_name = "snark-node" ; account_name = "snark-node-key" + ; docker_image = !Network.mina_image ; worker_nodes = 4 } ; txpool_max_size = 10_000_000 diff --git a/src/app/test_executive/payments_test.ml b/src/app/test_executive/payments_test.ml index aad1807f0f9..bd8dddeb472 100644 --- a/src/app/test_executive/payments_test.ml +++ b/src/app/test_executive/payments_test.ml @@ -18,40 +18,33 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Test_config in { default with genesis_ledger = - [ { account_name = "untimed-node-a-key" - ; balance = "400000" - ; timing = Untimed (* 400_000_000_000_000 *) - } - ; { account_name = "untimed-node-b-key" - ; balance = "300000" - ; timing = Untimed (* 300_000_000_000_000 *) - } - ; { account_name = "timed-node-c-key" - ; balance = "30000" - ; timing = - make_timing ~min_balance:10_000_000_000_000 ~cliff_time:8 - ~cliff_amount:0 ~vesting_period:4 - ~vesting_increment:5_000_000_000_000 - (* 30_000_000_000_000 mina is the total. initially, the balance will be 10k mina. after 8 global slots, the cliff is hit, although the cliff amount is 0. 4 slots after that, 5_000_000_000_000 mina will vest, and 4 slots after that another 5_000_000_000_000 will vest, and then twice again, for a total of 30k mina all fully liquid and unlocked at the end of the schedule*) - } - ; { account_name = "snark-node-key1"; balance = "0"; timing = Untimed } - ; { account_name = "snark-node-key2"; balance = "0"; timing = Untimed } - ; { account_name = "fish1"; balance = "100"; timing = Untimed } - ; { account_name = "fish2"; balance = "100"; timing = Untimed } + [ test_account "untimed-node-a-key" "400000" (* 400_000_000_000_000 *) + ; test_account "untimed-node-b-key" "300000" (* 300_000_000_000_000 *) + ; test_account "timed-node-c-key" "30000" + ~timing: + (make_timing ~min_balance:10_000_000_000_000 ~cliff_time:8 + ~cliff_amount:0 ~vesting_period:4 + ~vesting_increment:5_000_000_000_000 ) + (* 30_000_000_000_000 mina is the total. initially, the balance will be 10k mina. after 8 global slots, the cliff is hit, although the cliff amount is 0. 4 slots after that, 5_000_000_000_000 mina will vest, and 4 slots after that another 5_000_000_000_000 will vest, and then twice again, for a total of 30k mina all fully liquid and unlocked at the end of the schedule*) + ; test_account "snark-node-key1" "0" + ; test_account "snark-node-key2" "0" + ; test_account "fish1" "100" + ; test_account "fish2" "100" ] ; block_producers = - [ { node_name = "untimed-node-a"; account_name = "untimed-node-a-key" } - ; { node_name = "untimed-node-b"; account_name = "untimed-node-b-key" } - ; { node_name = "timed-node-c"; account_name = "timed-node-c-key" } + [ bp "untimed-node-a" !Network.mina_image + ; bp "untimed-node-b" !Network.mina_image + ; bp "timed-node-c" !Network.mina_image ] ; snark_coordinator = Some { node_name = "snark-node" ; account_name = "snark-node-key1" + ; docker_image = !Network.mina_image ; worker_nodes = 4 } - ; snark_worker_fee = "0.0002" - ; num_archive_nodes = 1 + ; snark_worker_fee = "0.0001" + ; archive_nodes = [] ; proof_config = { proof_config_default with work_delay = Some 1 diff --git a/src/app/test_executive/peers_reliability_test.ml b/src/app/test_executive/peers_reliability_test.ml index 98196416874..39d91e81c84 100644 --- a/src/app/test_executive/peers_reliability_test.ml +++ b/src/app/test_executive/peers_reliability_test.ml @@ -15,14 +15,14 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Test_config in { default with genesis_ledger = - [ { account_name = "node-a-key"; balance = "700000"; timing = Untimed } - ; { account_name = "node-b-key"; balance = "700000"; timing = Untimed } - ; { account_name = "node-c-key"; balance = "800000"; timing = Untimed } + [ test_account "node-a-key" "700000" + ; test_account "node-b-key" "700000" + ; test_account "node-c-key" "800000" ] ; block_producers = - [ { node_name = "node-a"; account_name = "node-a-key" } - ; { node_name = "node-b"; account_name = "node-b-key" } - ; { node_name = "node-c"; account_name = "node-c-key" } + [ bp "node-a" !Network.mina_image + ; bp "node-b" !Network.mina_image + ; bp "node-c" !Network.mina_image ] } diff --git a/src/app/test_executive/snarkyjs.ml b/src/app/test_executive/snarkyjs.ml index 9772b024c5c..63c148542f0 100644 --- a/src/app/test_executive/snarkyjs.ml +++ b/src/app/test_executive/snarkyjs.ml @@ -15,10 +15,8 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Test_config in { default with genesis_ledger = - [ { account_name = "node-key"; balance = "8000000"; timing = Untimed } - ; { account_name = "extra-key"; balance = "10"; timing = Untimed } - ] - ; block_producers = [ { node_name = "node"; account_name = "node-key" } ] + [ test_account "node-key" "8000000"; test_account "extra-key" "10" ] + ; block_producers = [ bp "node" !Network.mina_image ] } let check_and_print_stout_stderr ~logger process = diff --git a/src/app/test_executive/verification_key_update.ml b/src/app/test_executive/verification_key_update.ml index e741aa35396..981fcd8ce15 100644 --- a/src/app/test_executive/verification_key_update.ml +++ b/src/app/test_executive/verification_key_update.ml @@ -67,25 +67,18 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Test_config in { default with genesis_ledger = - [ { account_name = "whale1-key" - ; balance = "9000000000" - ; timing = Untimed - } - ; { account_name = "whale2-key" - ; balance = "1000000000" - ; timing = Untimed - } - ; { account_name = "snark-node-key"; balance = "100"; timing = Untimed } + [ test_account "whale1-key" "9000000000" + ; test_account "whale2-key" "1000000000" + ; test_account "snark-node-key" "100" ] ; block_producers = - [ { node_name = "whale1"; account_name = "whale1-key" } - ; { node_name = "whale2"; account_name = "whale2-key" } - ] - ; num_archive_nodes = 1 + [ bp "whale1" !Network.mina_image; bp "whale2" !Network.mina_image ] + ; archive_nodes = [] ; snark_coordinator = Some { node_name = "snark-node" ; account_name = "snark-node-key" + ; docker_image = !Network.mina_image ; worker_nodes = 2 } ; snark_worker_fee = "0.0001" diff --git a/src/app/test_executive/zkapps.ml b/src/app/test_executive/zkapps.ml index 32f816f1243..ba6db05a9bd 100644 --- a/src/app/test_executive/zkapps.ml +++ b/src/app/test_executive/zkapps.ml @@ -16,27 +16,20 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Test_config in { default with genesis_ledger = - [ { account_name = "node-a-key" - ; balance = "8000000000" - ; timing = Untimed - } - ; { account_name = "node-b-key" - ; balance = "1000000000" - ; timing = Untimed - } - ; { account_name = "fish1"; balance = "3000"; timing = Untimed } - ; { account_name = "fish2"; balance = "3000"; timing = Untimed } - ; { account_name = "snark-node-key"; balance = "0"; timing = Untimed } + [ test_account "node-a-key" "8000000000" + ; test_account "node-b-key" "1000000000" + ; test_account "fish1" "3000" + ; test_account "fish2" "3000" + ; test_account "snark-node-key" "0" ] ; block_producers = - [ { node_name = "node-a"; account_name = "node-a-key" } - ; { node_name = "node-b"; account_name = "node-b-key" } - ] - ; num_archive_nodes = 1 + [ bp "node-a" !Network.mina_image; bp "node-b" !Network.mina_image ] + ; archive_nodes = [] ; snark_coordinator = Some { node_name = "snark-node" ; account_name = "snark-node-key" + ; docker_image = !Network.mina_image ; worker_nodes = 2 } ; snark_worker_fee = "0.0001" diff --git a/src/app/test_executive/zkapps_nonce_test.ml b/src/app/test_executive/zkapps_nonce_test.ml index 7ff563285ca..c175b648278 100644 --- a/src/app/test_executive/zkapps_nonce_test.ml +++ b/src/app/test_executive/zkapps_nonce_test.ml @@ -16,24 +16,20 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Test_config in { default with genesis_ledger = - [ { account_name = "node-a-key" - ; balance = "8000000000" - ; timing = Untimed - } - ; { account_name = "node-b-key"; balance = "1000000"; timing = Untimed } - ; { account_name = "fish1"; balance = "3000"; timing = Untimed } - ; { account_name = "fish2"; balance = "3000"; timing = Untimed } - ; { account_name = "snark-node-key"; balance = "0"; timing = Untimed } + [ test_account "node-a-key" "8000000000" + ; test_account "node-b-key" "1000000" + ; test_account "fish1" "3000" + ; test_account "fish2" "3000" + ; test_account "snark-node-key" "0" ] ; block_producers = - [ { node_name = "node-a"; account_name = "node-a-key" } - ; { node_name = "node-b"; account_name = "node-b-key" } - ] - ; num_archive_nodes = 1 + [ bp "node-a" !Network.mina_image; bp "node-b" !Network.mina_image ] + ; archive_nodes = [] ; snark_coordinator = Some { node_name = "snark-node" ; account_name = "snark-node-key" + ; docker_image = !Network.mina_image ; worker_nodes = 5 } ; snark_worker_fee = "0.0001" diff --git a/src/app/test_executive/zkapps_timing.ml b/src/app/test_executive/zkapps_timing.ml index c763117e6df..f513a60a818 100644 --- a/src/app/test_executive/zkapps_timing.ml +++ b/src/app/test_executive/zkapps_timing.ml @@ -15,25 +15,16 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let open Test_config in { default with genesis_ledger = - [ { account_name = "node-a-key" - ; balance = "8000000000" - ; timing = Untimed - } - ; { account_name = "node-b-key" - ; balance = "1000000000" - ; timing = Untimed - } - ; { account_name = "node-c-key" - ; balance = "1000000000" - ; timing = Untimed - } + [ test_account "node-a-key" "8000000000" + ; test_account "node-b-key" "1000000000" + ; test_account "node-c-key" "1000000000" ] ; block_producers = - [ { node_name = "node-a"; account_name = "node-a-key" } - ; { node_name = "node-b"; account_name = "node-b-key" } - ; { node_name = "node-c"; account_name = "node-c-key" } + [ bp "node-a" !Network.mina_image + ; bp "node-b" !Network.mina_image + ; bp "node-c" !Network.mina_image ] - ; num_archive_nodes = 1 + ; archive_nodes = [] } let run network t = From c0ed6beb8139efe0a42748a170922f84b5437777 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Fri, 1 Sep 2023 22:39:53 +0000 Subject: [PATCH 48/90] (intg test) Rename Ci_interaction -> Config_util --- .../integration_test_abstract_engine/abstract_network.ml | 3 +-- .../{ci_interaction.ml => config_util.ml} | 7 +++++-- .../integration_test_abstract_engine.ml | 2 +- .../integration_test_abstract_engine/mina_automation.ml | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) rename src/lib/integration_test_abstract_engine/{ci_interaction.ml => config_util.ml} (99%) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index f33c82883e5..34cb1ed3b45 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -2,7 +2,7 @@ open Core_kernel open Async open Integration_test_lib open Mina_transaction -open Ci_interaction +open Config_util (* exclude from bisect_ppx to avoid type error on GraphQL modules *) [@@@coverage exclude_file] @@ -1290,7 +1290,6 @@ let all_node_id t = let pods = all_pods t |> Core.Map.to_alist in List.fold pods ~init:[] ~f:(fun acc (_, node) -> node.node_id :: acc) -(* TODO: expand or remove *) let initialize_infra ~logger network = [%log trace] "initialize_infra: %s" network.network_id ; Malleable_error.return () diff --git a/src/lib/integration_test_abstract_engine/ci_interaction.ml b/src/lib/integration_test_abstract_engine/config_util.ml similarity index 99% rename from src/lib/integration_test_abstract_engine/ci_interaction.ml rename to src/lib/integration_test_abstract_engine/config_util.ml index 286ec354704..a31add08556 100644 --- a/src/lib/integration_test_abstract_engine/ci_interaction.ml +++ b/src/lib/integration_test_abstract_engine/config_util.ml @@ -656,7 +656,7 @@ module Config_file = struct exception Missing_arg of string * string * string - exception Invalid_arg_type of string * Yojson.Safe.t * string + exception Invalid_arg_type of string * string * Yojson.Safe.t * string let version config = match Yojson.Safe.Util.member "version" config with @@ -726,7 +726,10 @@ module Config_file = struct raise @@ Missing_arg (action_name action, arg_name, arg_type) in if validate_arg_type arg_type arg_value then aux rest - else raise @@ Invalid_arg_type (arg_name, arg_value, arg_type) + else + raise + @@ Invalid_arg_type + (action_name action, arg_name, arg_value, arg_type) in aux arg_list diff --git a/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.ml b/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.ml index 37e989045fc..0c42086870e 100644 --- a/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.ml +++ b/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.ml @@ -1,6 +1,6 @@ let name = "abstract" module Network = Abstract_network -module Network_config = Ci_interaction.Network_config +module Network_config = Config_util.Network_config module Network_manager = Mina_automation.Network_manager module Log_engine = Graphql_polling_log_engine diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index 35b82b19324..e9ef61e9b72 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -2,7 +2,7 @@ open Core open Core_unix open Async open Integration_test_lib -open Ci_interaction +open Config_util module Network_manager = struct type t = From ca48889067e859d019ef79fc280c14d1c7b18ff4 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Fri, 1 Sep 2023 22:43:49 +0000 Subject: [PATCH 49/90] (unit test) Sync config file --- .../test_data/config.json | 5 ++--- .../unit_tests.ml | 20 ++++++++----------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/test_data/config.json b/src/lib/integration_test_abstract_engine/test_data/config.json index 06951e3f21b..4c33235bf99 100644 --- a/src/lib/integration_test_abstract_engine/test_data/config.json +++ b/src/lib/integration_test_abstract_engine/test_data/config.json @@ -32,10 +32,9 @@ "args": { "network_id":"string", "node_id":"string", - "fresh_state":"bool", - "git_commit":"string" + "fresh_state":"bool" }, - "command": "minimina node start --network-id {{network_id}} --node-id {{node_id}} {{fresh_state}} --git-commit {{git_commit}}" + "command": "minimina node start --network-id {{network_id}} --node-id {{node_id}} {{fresh_state}}" }, { "name": "stop_node", diff --git a/src/lib/integration_test_abstract_engine/unit_tests.ml b/src/lib/integration_test_abstract_engine/unit_tests.ml index 73c486f9b8b..954111f00f9 100644 --- a/src/lib/integration_test_abstract_engine/unit_tests.ml +++ b/src/lib/integration_test_abstract_engine/unit_tests.ml @@ -1,6 +1,6 @@ open Core_kernel open Integration_test_lib -open Ci_interaction +open Config_util module Test_values = struct let config_file = @@ -17,7 +17,7 @@ module Test_values = struct let start_node_raw_cmd = "minimina node start --network-id {{network_id}} --node-id {{node_id}} \ - {{fresh_state}} --git-commit {{git_commit}}" + {{fresh_state}}" let deploy_network_action = sprintf @@ -53,8 +53,7 @@ module Test_values = struct "args": { "network_id": "string", "node_id": "string", - "fresh_state": "bool", - "git_commit": "string" + "fresh_state": "bool" }, "command": "%s" } @@ -139,7 +138,6 @@ module Config_tests = struct [ ("network_id", `String "network0") ; ("node_id", `String "node0") ; ("fresh_state", `Bool true) - ; ("git_commit", `String "0123456abcdef") ] in assert (validate_args ~args ~action) @@ -157,10 +155,8 @@ module Config_tests = struct let action = Yojson.Safe.from_string start_node_action in let node_id = "node0" in let network_id = "network0" in - let git_commit = "0123456abcdef" in let args = [ ("fresh_state", `Bool true) - ; ("git_commit", `String git_commit) ; ("network_id", `String network_id) ; ("node_id", `String node_id) ] @@ -175,8 +171,6 @@ module Config_tests = struct ; "--node-id" ; node_id ; "--fresh-state" - ; "--git-commit" - ; git_commit ] in assert (prog = List.hd_exn expect) ; @@ -196,9 +190,10 @@ module Run_command_tests = struct @@ run_command ~suppress_logs:true ~config:config_file ~args:[ (arg, value) ] "echo" - with Invalid_arg_type (arg_name, arg_value, arg_type) -> + with Invalid_arg_type (action_name, arg_name, arg_value, arg_type) -> assert ( - arg_name = "msg" && arg_type = Arg_type.string + action_name = "\"echo\"" && arg_name = "msg" + && arg_type = Arg_type.string && Yojson.Safe.equal arg_value value ) let%test_unit "run command arg number failure" = @@ -222,7 +217,8 @@ module Run_command_tests = struct "echo" with Missing_arg (action_name, arg_name, arg_type) -> assert ( - action_name = "echo" && arg_name = "msg" && arg_type = Arg_type.string ) + action_name = "\"echo\"" && arg_name = "msg" + && arg_type = Arg_type.string ) end module Successful = struct From 50eeb0299e160e13266943604d12f96bf903757d Mon Sep 17 00:00:00 2001 From: Quantifier Date: Fri, 1 Sep 2023 23:29:53 +0000 Subject: [PATCH 50/90] (intg test) Use pre-generated keypairs in all engines --- .../config_util.ml | 41 +------------------ .../mina_automation.ml | 19 +-------- src/lib/integration_test_lib/util.ml | 40 ++++++++++++++++++ 3 files changed, 42 insertions(+), 58 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/config_util.ml b/src/lib/integration_test_abstract_engine/config_util.ml index a31add08556..bff18978422 100644 --- a/src/lib/integration_test_abstract_engine/config_util.ml +++ b/src/lib/integration_test_abstract_engine/config_util.ml @@ -70,45 +70,6 @@ module Network_config = struct let testnet_log_filter t = t.config.network_id - let pull_keypairs num_keypairs = - let int_list = List.range ~stop:`inclusive 1 10_000 in - let random_nums = - Quickcheck.Generator.(of_list int_list |> list_with_length num_keypairs) - |> Quickcheck.random_value - in - let normalize_path path = - if String.(suffix path 1 = "/") then String.drop_suffix path 1 else path - in - (* network keypairs + private keys *) - let keypairs_path = normalize_path !keypairs_path in - let base_filename n = - sprintf "%s/network-keypairs/sender-account-%d.json" keypairs_path n - in - let read_sk n = In_channel.read_all (base_filename n ^ ".sk") in - let read_keypair n = - let open Yojson.Safe in - let json = from_file (base_filename n) in - let sk = read_sk n |> Private_key.of_base58_check_exn in - { Network_keypair.keypair = Keypair.of_private_key_exn sk - ; keypair_name = Util.member "keypair_name" json |> to_string - ; privkey_password = Util.member "privkey_password" json |> to_string - ; public_key = Util.member "public_key" json |> to_string - ; private_key = Util.member "private_key" json |> to_string - } - in - (* libp2p keypairs *) - let libp2p_base_filename n = - sprintf "%s/libp2p-keypairs/libp2p-keypair-%d.json" keypairs_path n - in - let read_peerid n = - In_channel.read_all (libp2p_base_filename n ^ ".peerid") - in - let read_libp2p n = Yojson.Safe.from_file (libp2p_base_filename n) in - ( List.map random_nums ~f:read_keypair - , List.map random_nums ~f:read_sk - , List.map random_nums ~f:read_libp2p - , List.map random_nums ~f:read_peerid ) - let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) = @@ -157,7 +118,7 @@ module Network_config = struct let before = Time.now () in let num_keypairs = List.length genesis_ledger in let network_keypairs, private_keys, _libp2p_keypairs, _libp2p_peerids = - pull_keypairs (List.length genesis_ledger) + Util.pull_keypairs !keypairs_path (List.length genesis_ledger) in [%log trace] "Pulled %d keypairs from %s in %s" num_keypairs !keypairs_path Time.(abs_diff before @@ now () |> Span.to_string) ; diff --git a/src/lib/integration_test_cloud_engine/mina_automation.ml b/src/lib/integration_test_cloud_engine/mina_automation.ml index 72388dea5ea..b2f997f00d2 100644 --- a/src/lib/integration_test_cloud_engine/mina_automation.ml +++ b/src/lib/integration_test_cloud_engine/mina_automation.ml @@ -115,9 +115,7 @@ module Network_config = struct ; epoch_data ; archive_nodes ; block_producers - ; seed_nodes ; snark_coordinator - ; snark_workers ; snark_worker_fee ; log_precomputed_blocks (* ; num_plain_nodes *) ; proof_config @@ -126,18 +124,10 @@ module Network_config = struct ; slots_per_epoch ; slots_per_sub_window ; txpool_max_size + ; _ } = test_config in - List.iter archive_nodes ~f:(fun x -> - Test_config.Archive_node.to_yojson x - |> Yojson.Safe.pretty_to_string |> print_endline ) ; - List.iter seed_nodes ~f:(fun x -> - Test_config.Seed_node.to_yojson x - |> Yojson.Safe.pretty_to_string |> print_endline ) ; - List.iter snark_workers ~f:(fun x -> - Test_config.Snark_worker_node.to_yojson x - |> Yojson.Safe.pretty_to_string |> print_endline ) ; let user_from_env = Option.value (Unix.getenv "USER") ~default:"auto" in let user_sanitized = Str.global_replace (Str.regexp "\\W|_-") "" user_from_env @@ -352,12 +342,6 @@ module Network_config = struct { constraints = constraint_constants; genesis = genesis_constants } in (* BLOCK PRODUCER CONFIG *) - let mk_net_keypair keypair_name (pk, sk) = - let keypair = - { Keypair.public_key = Public_key.decompress_exn pk; private_key = sk } - in - Network_keypair.create_network_keypair ~keypair_name ~keypair - in let block_producer_config name keypair = { name; keypair; libp2p_secret = "" } in @@ -433,7 +417,6 @@ module Network_config = struct ; worker_nodes = node.worker_nodes } in - (* NETWORK CONFIG *) { mina_automation_location = cli_inputs.mina_automation_location ; debug_arg = debug diff --git a/src/lib/integration_test_lib/util.ml b/src/lib/integration_test_lib/util.ml index 341ee4eb8e0..10b226ce591 100644 --- a/src/lib/integration_test_lib/util.ml +++ b/src/lib/integration_test_lib/util.ml @@ -23,6 +23,46 @@ let drop_outer_quotes s = let s = drop_suffix s n in s +let pull_keypairs path num_keypairs = + let int_list = List.range ~stop:`inclusive 1 10_000 in + let random_nums = + Quickcheck.Generator.(of_list int_list |> list_with_length num_keypairs) + |> Quickcheck.random_value + in + let normalize_path path = + if String.(suffix path 1 = "/") then String.drop_suffix path 1 else path + in + (* network keypairs + private keys *) + let keypairs_path = normalize_path path in + let base_filename n = + sprintf "%s/network-keypairs/sender-account-%d.json" keypairs_path n + in + let read_sk n = In_channel.read_all (base_filename n ^ ".sk") in + let read_keypair n = + let open Yojson.Safe in + let open Signature_lib in + let json = from_file (base_filename n) in + let sk = read_sk n |> Private_key.of_base58_check_exn in + { Network_keypair.keypair = Keypair.of_private_key_exn sk + ; keypair_name = Util.member "keypair_name" json |> to_string + ; privkey_password = Util.member "privkey_password" json |> to_string + ; public_key = Util.member "public_key" json |> to_string + ; private_key = Util.member "private_key" json |> to_string + } + in + (* libp2p keypairs *) + let libp2p_base_filename n = + sprintf "%s/libp2p-keypairs/libp2p-keypair-%d.json" keypairs_path n + in + let read_peerid n = + In_channel.read_all (libp2p_base_filename n ^ ".peerid") + in + let read_libp2p n = Yojson.Safe.from_file (libp2p_base_filename n) in + ( List.map random_nums ~f:read_keypair + , List.map random_nums ~f:read_sk + , List.map random_nums ~f:read_libp2p + , List.map random_nums ~f:read_peerid ) + let check_cmd_output ~prog ~args output = let open Process.Output in let print_output () = From 47ba18ba6ed343359438fcc9c0b3940cba103c21 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sun, 3 Sep 2023 00:00:18 +0000 Subject: [PATCH 51/90] (test exec) Cleanup after network setup in mock test --- src/app/test_executive/test_executive.ml | 80 +++++++++++++----------- 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index 240d8b7046f..8bad3986c6b 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -400,44 +400,50 @@ let main inputs = in [%log trace] "initializing network abstraction" ; let%bind () = Engine.Network.initialize_infra ~logger network in - [%log info] "starting the daemons within the pods" ; - let start_print (node : Engine.Network.Node.t) = - let open Malleable_error.Let_syntax in - [%log info] "starting %s..." (Engine.Network.Node.id node) ; - let%bind res = Engine.Network.Node.start ~fresh_state:false node in - [%log info] "%s started" (Engine.Network.Node.id node) ; - Malleable_error.return res - in - let seed_nodes = - network |> Engine.Network.seeds |> Core.String.Map.data - in - let non_seed_pods = - network |> Engine.Network.all_non_seed_pods |> Core.String.Map.data - in - let _offline_node_event_subscription = - (* Monitor for offline nodes; abort the test if a node goes down - unexpectedly. - *) - Dsl.Event_router.on (Dsl.event_router dsl) Node_offline - ~f:(fun offline_node () -> - let node_name = Engine.Network.Node.app_id offline_node in - [%log info] "Detected node offline $node" - ~metadata:[ ("node", `String node_name) ] ; - if Engine.Network.Node.should_be_running offline_node then ( - [%log fatal] "Offline $node is required for this test" + if String.equal test_name "mock" then ( + [%log info] "No node interactions in mock network" ; + return () ) + else ( + [%log info] "starting the daemons within the pods" ; + let start_print (node : Engine.Network.Node.t) = + let open Malleable_error.Let_syntax in + [%log info] "starting %s..." (Engine.Network.Node.id node) ; + let%bind res = Engine.Network.Node.start ~fresh_state:false node in + [%log info] "%s started" (Engine.Network.Node.id node) ; + Malleable_error.return res + in + let seed_nodes = + network |> Engine.Network.seeds |> Core.String.Map.data + in + let non_seed_pods = + network |> Engine.Network.all_non_seed_pods |> Core.String.Map.data + in + let _offline_node_event_subscription = + (* Monitor for offline nodes; abort the test if a node goes down + unexpectedly. + *) + Dsl.Event_router.on (Dsl.event_router dsl) Node_offline + ~f:(fun offline_node () -> + let node_name = Engine.Network.Node.app_id offline_node in + [%log info] "Detected node offline $node" ~metadata:[ ("node", `String node_name) ] ; - failwith "Aborted because of required offline node" ) ; - Async_kernel.Deferred.return `Continue ) - in - (* TODO: parallelize (requires accumlative hard errors) *) - let%bind () = Malleable_error.List.iter seed_nodes ~f:start_print in - let%bind () = - Dsl.(wait_for dsl @@ Wait_condition.nodes_to_initialize seed_nodes) - in - let%bind () = Malleable_error.List.iter non_seed_pods ~f:start_print in - [%log info] "daemons started" ; - [%log trace] "executing test" ; - T.run network dsl ) + if Engine.Network.Node.should_be_running offline_node then ( + [%log fatal] "Offline $node is required for this test" + ~metadata:[ ("node", `String node_name) ] ; + failwith "Aborted because of required offline node" ) ; + Async_kernel.Deferred.return `Continue ) + in + (* TODO: parallelize (requires accumlative hard errors) *) + let%bind () = Malleable_error.List.iter seed_nodes ~f:start_print in + let%bind () = + Dsl.(wait_for dsl @@ Wait_condition.nodes_to_initialize seed_nodes) + in + let%bind () = + Malleable_error.List.iter non_seed_pods ~f:start_print + in + [%log info] "daemons started" ; + [%log trace] "executing test" ; + T.run network dsl ) ) in let exit_reason, test_result = match monitor_test_result with From 0959a28aa9219f4730f395600274902d9c1aa0ce Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sun, 3 Sep 2023 00:22:38 +0000 Subject: [PATCH 52/90] (intg test) Separate abstract engine network config --- .../config_util.ml | 310 +----------------- .../integration_test_abstract_engine.ml | 2 +- .../network_config.ml | 307 +++++++++++++++++ 3 files changed, 310 insertions(+), 309 deletions(-) create mode 100644 src/lib/integration_test_abstract_engine/network_config.ml diff --git a/src/lib/integration_test_abstract_engine/config_util.ml b/src/lib/integration_test_abstract_engine/config_util.ml index bff18978422..75ccde61d20 100644 --- a/src/lib/integration_test_abstract_engine/config_util.ml +++ b/src/lib/integration_test_abstract_engine/config_util.ml @@ -1,9 +1,8 @@ open Core_kernel open Async -open Currency open Signature_lib -open Mina_base open Integration_test_lib +module Network_config = Network_config (* [alias] is instantiated when command line args are parsed *) let alias = ref None @@ -14,315 +13,10 @@ let archive_image : string option ref = ref None (* [config_path] is instantiated when command line args are parsed *) let config_path = ref "" -(* [keypairs_path] is instantiated when command line args are parsed *) -let keypairs_path = ref "" - (* [mina_image] is instantiated when command line args are parsed *) let mina_image = ref "" -module Network_config = struct - module Cli_inputs = Cli_inputs - - type block_producer_config = - { name : string; keypair : Network_keypair.t; libp2p_secret : string } - [@@deriving to_yojson] - - type snark_coordinator_config = - { name : string; public_key : string; worker_nodes : int } - [@@deriving to_yojson] - - type config = - { network_id : string - ; config_dir : string - ; deploy_graphql_ingress : bool - ; mina_image : string - ; mina_agent_image : string - ; mina_bots_image : string - ; mina_points_image : string - ; mina_archive_image : string - ; runtime_config : Yojson.Safe.t - [@to_yojson fun j -> `String (Yojson.Safe.to_string j)] - ; block_producer_configs : block_producer_config list - ; log_precomputed_blocks : bool - ; archive_node_count : int - ; mina_archive_schema : string - ; mina_archive_schema_aux_files : string list - ; snark_coordinator_config : snark_coordinator_config option - ; snark_worker_fee : string - } - [@@deriving to_yojson] - - type t = - { debug_arg : bool - ; genesis_keypairs : - (Network_keypair.t Core.String.Map.t - [@to_yojson - fun map -> - `Assoc - (Core.Map.fold_right ~init:[] - ~f:(fun ~key:k ~data:v accum -> - (k, Network_keypair.to_yojson v) :: accum ) - map )] ) - ; constants : Test_config.constants - ; config : config - } - [@@deriving to_yojson] - - let testnet_log_filter t = t.config.network_id - - let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) - ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) - = - let { requires_graphql - ; genesis_ledger - ; archive_nodes - ; block_producers - ; snark_coordinator - ; snark_worker_fee - ; log_precomputed_blocks - ; proof_config - ; Test_config.k - ; delta - ; slots_per_epoch - ; slots_per_sub_window - ; txpool_max_size - ; _ - } = - test_config - in - let user_from_env = Option.value (Unix.getenv "USER") ~default:"auto" in - let user_sanitized = - Str.global_replace (Str.regexp "\\W|_-") "" user_from_env - in - let user_len = Int.min 5 (String.length user_sanitized) in - let user = String.sub user_sanitized ~pos:0 ~len:user_len in - let git_commit = Mina_version.commit_id_short in - let network_id = sprintf "it-%s-%s-%s" user git_commit test_name in - (* check to make sure the test writer hasn't accidentally created duplicate names of accounts and keys *) - let key_names_list = - List.map genesis_ledger ~f:(fun acct -> acct.account_name) - in - if List.contains_dup ~compare:String.compare key_names_list then - failwith - "All accounts in genesis ledger must have unique names. Check to make \ - sure you are not using the same account_name more than once" ; - let all_nodes_names_list = - List.map block_producers ~f:(fun acct -> acct.node_name) - @ match snark_coordinator with None -> [] | Some n -> [ n.node_name ] - in - if List.contains_dup ~compare:String.compare all_nodes_names_list then - failwith - "All nodes in testnet must have unique names. Check to make sure you \ - are not using the same node_name more than once" ; - (* ACCOUNTS AND KEYPAIRS *) - let before = Time.now () in - let num_keypairs = List.length genesis_ledger in - let network_keypairs, private_keys, _libp2p_keypairs, _libp2p_peerids = - Util.pull_keypairs !keypairs_path (List.length genesis_ledger) - in - [%log trace] "Pulled %d keypairs from %s in %s" num_keypairs !keypairs_path - Time.(abs_diff before @@ now () |> Span.to_string) ; - let labeled_accounts : - (Runtime_config.Accounts.single * (Network_keypair.t * Private_key.t)) - String.Map.t = - String.Map.empty - in - let rec add_accounts acc = function - | [] -> - acc - | hd :: tl -> - let ( { Test_config.Test_Account.balance; account_name; timing; _ } - , (network_keypair, sk) ) = - hd - in - let timing = - match timing with - | Account.Timing.Untimed -> - None - | Timed t -> - Some - { Runtime_config.Accounts.Single.Timed.initial_minimum_balance = - t.initial_minimum_balance - ; cliff_time = t.cliff_time - ; cliff_amount = t.cliff_amount - ; vesting_period = t.vesting_period - ; vesting_increment = t.vesting_increment - } - in - let default = Runtime_config.Accounts.Single.default in - let sk = Private_key.of_base58_check_exn sk in - let pk = Public_key.(of_private_key_exn sk |> compress) in - let acct = - { default with - pk = Public_key.Compressed.to_string pk - ; sk = Some (Private_key.to_base58_check sk) - ; balance = - Balance.of_mina_string_exn balance - (* delegation currently unsupported *) - ; delegate = None - ; timing - } - in - add_accounts - (String.Map.add_exn acc ~key:account_name - ~data:(acct, (network_keypair, sk)) ) - tl - in - let genesis_ledger_accounts = - let network_keypairs = - List.mapi network_keypairs ~f:(fun n kp -> - { kp with - keypair_name = (List.nth_exn genesis_ledger n).account_name - } ) - in - let keypairs = List.zip_exn network_keypairs private_keys in - add_accounts labeled_accounts (List.zip_exn genesis_ledger keypairs) - in - (* DAEMON CONFIG *) - let constraint_constants = - Genesis_ledger_helper.make_constraint_constants - ~default:Genesis_constants.Constraint_constants.compiled proof_config - in - let runtime_config = - { Runtime_config.daemon = - Some - { txpool_max_size = Some txpool_max_size - ; peer_list_url = None - ; zkapp_proof_update_cost = None - ; zkapp_signed_single_update_cost = None - ; zkapp_signed_pair_update_cost = None - ; zkapp_transaction_cost_limit = None - ; max_event_elements = None - ; max_action_elements = None - } - ; genesis = - Some - { k = Some k - ; delta = Some delta - ; slots_per_epoch = Some slots_per_epoch - ; slots_per_sub_window = Some slots_per_sub_window - ; genesis_state_timestamp = - Some Core.Time.(to_string_abs ~zone:Zone.utc @@ now ()) - } - ; proof = Some proof_config (* TODO: prebake ledger and only set hash *) - ; ledger = - Some - { base = - Accounts - (String.Map.data genesis_ledger_accounts |> List.map ~f:fst) - ; add_genesis_winner = None - ; num_accounts = None - ; balances = [] - ; hash = None - ; name = None - } - ; epoch_data = None - } - in - let genesis_constants = - Or_error.ok_exn - (Genesis_ledger_helper.make_genesis_constants ~logger - ~default:Genesis_constants.compiled runtime_config ) - in - let constants : Test_config.constants = - { constraints = constraint_constants; genesis = genesis_constants } - in - (* BLOCK PRODUCER CONFIG *) - let block_producer_config name keypair = - { name; keypair; libp2p_secret = "" } - in - let block_producer_configs = - List.mapi block_producers ~f:(fun n node -> - let _ = - match String.Map.find genesis_ledger_accounts node.account_name with - | Some acct -> - acct - | None -> - let failstring = - Format.sprintf - "Failing because the account key of all initial block \ - producers must be in the genesis ledger. name of Node: \ - %s. name of Account which does not exist: %s" - node.node_name node.account_name - in - failwith failstring - in - block_producer_config node.node_name - @@ List.nth_exn network_keypairs n ) - in - let mina_archive_schema = "create_schema.sql" in - let long_commit_id = - let open String in - let id = Mina_version.commit_id in - if prefix id 7 = "[DIRTY]" then suffix id (length id - 7) else id - in - let mina_archive_base_url = - "https://raw.githubusercontent.com/MinaProtocol/mina/" ^ long_commit_id - ^ "/src/app/archive/" - in - let mina_archive_schema_aux_files = - [ mina_archive_base_url ^ "create_schema.sql" - ; mina_archive_base_url ^ "zkapp_tables.sql" - ] - in - let genesis_keypairs = - String.Map.of_alist_exn - (List.map (String.Map.to_alist genesis_ledger_accounts) - ~f:(fun element -> - let kp_name, (_, (keypair, _)) = element in - (kp_name, keypair) ) ) - in - let snark_coordinator_config = - match snark_coordinator with - | None -> - None - | Some node -> - let network_kp = - match String.Map.find genesis_keypairs node.account_name with - | Some acct -> - acct - | None -> - let failstring = - Format.sprintf - "Failing because the account key of all initial snark \ - coordinators must be in the genesis ledger. name of \ - Node: %s. name of Account which does not exist: %s" - node.node_name node.account_name - in - failwith failstring - in - Some - { name = node.node_name - ; public_key = - Public_key.Compressed.to_base58_check - (Public_key.compress network_kp.keypair.public_key) - ; worker_nodes = node.worker_nodes - } - in - (* NETWORK CONFIG *) - { debug_arg = debug - ; genesis_keypairs - ; constants - ; config = - { network_id - ; config_dir = cli_inputs.mina_automation_location - ; deploy_graphql_ingress = requires_graphql - ; mina_image = images.mina - ; mina_agent_image = images.user_agent - ; mina_bots_image = images.bots - ; mina_points_image = images.points - ; mina_archive_image = images.archive_node - ; runtime_config = Runtime_config.to_yojson runtime_config - ; block_producer_configs - ; log_precomputed_blocks - ; archive_node_count = List.length archive_nodes - ; mina_archive_schema - ; mina_archive_schema_aux_files - ; snark_coordinator_config - ; snark_worker_fee - } - } -end +let keypairs_path = Network_config.keypairs_path module Network_id = struct type t = string [@@deriving eq, yojson] diff --git a/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.ml b/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.ml index 0c42086870e..dcb0c014157 100644 --- a/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.ml +++ b/src/lib/integration_test_abstract_engine/integration_test_abstract_engine.ml @@ -1,6 +1,6 @@ let name = "abstract" module Network = Abstract_network -module Network_config = Config_util.Network_config +module Network_config = Network_config module Network_manager = Mina_automation.Network_manager module Log_engine = Graphql_polling_log_engine diff --git a/src/lib/integration_test_abstract_engine/network_config.ml b/src/lib/integration_test_abstract_engine/network_config.ml new file mode 100644 index 00000000000..1890c1a098d --- /dev/null +++ b/src/lib/integration_test_abstract_engine/network_config.ml @@ -0,0 +1,307 @@ +open Core_kernel +open Async +open Currency +open Signature_lib +open Mina_base +open Integration_test_lib +module Cli_inputs = Cli_inputs + +(* [keypairs_path] is instantiated when command line args are parsed *) +let keypairs_path = ref "" + +type block_producer_config = + { name : string; keypair : Network_keypair.t; libp2p_secret : string } +[@@deriving to_yojson] + +type snark_coordinator_config = + { name : string; public_key : string; worker_nodes : int } +[@@deriving to_yojson] + +type config = + { network_id : string + ; config_dir : string + ; deploy_graphql_ingress : bool + ; mina_image : string + ; mina_agent_image : string + ; mina_bots_image : string + ; mina_points_image : string + ; mina_archive_image : string + ; runtime_config : Yojson.Safe.t + [@to_yojson fun j -> `String (Yojson.Safe.to_string j)] + ; block_producer_configs : block_producer_config list + ; log_precomputed_blocks : bool + ; archive_node_count : int + ; mina_archive_schema : string + ; mina_archive_schema_aux_files : string list + ; snark_coordinator_config : snark_coordinator_config option + ; snark_worker_fee : string + } +[@@deriving to_yojson] + +type t = + { debug_arg : bool + ; genesis_keypairs : + (Network_keypair.t Core.String.Map.t + [@to_yojson + fun map -> + `Assoc + (Core.Map.fold_right ~init:[] + ~f:(fun ~key:k ~data:v accum -> + (k, Network_keypair.to_yojson v) :: accum ) + map )] ) + ; constants : Test_config.constants + ; config : config + } +[@@deriving to_yojson] + +let testnet_log_filter t = t.config.network_id + +let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) + ~(test_config : Test_config.t) ~(images : Test_config.Container_images.t) = + let { requires_graphql + ; genesis_ledger + ; archive_nodes + ; block_producers + ; snark_coordinator + ; snark_worker_fee + ; log_precomputed_blocks + ; proof_config + ; Test_config.k + ; delta + ; slots_per_epoch + ; slots_per_sub_window + ; txpool_max_size + ; _ + } = + test_config + in + let user_from_env = Option.value (Unix.getenv "USER") ~default:"auto" in + let user_sanitized = + Str.global_replace (Str.regexp "\\W|_-") "" user_from_env + in + let user_len = Int.min 5 (String.length user_sanitized) in + let user = String.sub user_sanitized ~pos:0 ~len:user_len in + let git_commit = Mina_version.commit_id_short in + let network_id = sprintf "it-%s-%s-%s" user git_commit test_name in + (* check to make sure the test writer hasn't accidentally created duplicate names of accounts and keys *) + let key_names_list = + List.map genesis_ledger ~f:(fun acct -> acct.account_name) + in + if List.contains_dup ~compare:String.compare key_names_list then + failwith + "All accounts in genesis ledger must have unique names. Check to make \ + sure you are not using the same account_name more than once" ; + let all_nodes_names_list = + List.map block_producers ~f:(fun acct -> acct.node_name) + @ match snark_coordinator with None -> [] | Some n -> [ n.node_name ] + in + if List.contains_dup ~compare:String.compare all_nodes_names_list then + failwith + "All nodes in testnet must have unique names. Check to make sure you \ + are not using the same node_name more than once" ; + (* ACCOUNTS AND KEYPAIRS *) + let before = Time.now () in + let num_keypairs = List.length genesis_ledger in + let network_keypairs, private_keys, _libp2p_keypairs, _libp2p_peerids = + Util.pull_keypairs !keypairs_path (List.length genesis_ledger) + in + [%log trace] "Pulled %d keypairs from %s in %s" num_keypairs !keypairs_path + Time.(abs_diff before @@ now () |> Span.to_string) ; + let labeled_accounts : + (Runtime_config.Accounts.single * (Network_keypair.t * Private_key.t)) + String.Map.t = + String.Map.empty + in + let rec add_accounts acc = function + | [] -> + acc + | hd :: tl -> + let ( { Test_config.Test_Account.balance; account_name; timing; _ } + , (network_keypair, sk) ) = + hd + in + let timing = + match timing with + | Account.Timing.Untimed -> + None + | Timed t -> + Some + { Runtime_config.Accounts.Single.Timed.initial_minimum_balance = + t.initial_minimum_balance + ; cliff_time = t.cliff_time + ; cliff_amount = t.cliff_amount + ; vesting_period = t.vesting_period + ; vesting_increment = t.vesting_increment + } + in + let default = Runtime_config.Accounts.Single.default in + let sk = Private_key.of_base58_check_exn sk in + let pk = Public_key.(of_private_key_exn sk |> compress) in + let acct = + { default with + pk = Public_key.Compressed.to_string pk + ; sk = Some (Private_key.to_base58_check sk) + ; balance = + Balance.of_mina_string_exn balance + (* delegation currently unsupported *) + ; delegate = None + ; timing + } + in + add_accounts + (String.Map.add_exn acc ~key:account_name + ~data:(acct, (network_keypair, sk)) ) + tl + in + let genesis_ledger_accounts = + let network_keypairs = + List.mapi network_keypairs ~f:(fun n kp -> + { kp with + keypair_name = (List.nth_exn genesis_ledger n).account_name + } ) + in + let keypairs = List.zip_exn network_keypairs private_keys in + add_accounts labeled_accounts (List.zip_exn genesis_ledger keypairs) + in + (* DAEMON CONFIG *) + let constraint_constants = + Genesis_ledger_helper.make_constraint_constants + ~default:Genesis_constants.Constraint_constants.compiled proof_config + in + let runtime_config = + { Runtime_config.daemon = + Some + { txpool_max_size = Some txpool_max_size + ; peer_list_url = None + ; zkapp_proof_update_cost = None + ; zkapp_signed_single_update_cost = None + ; zkapp_signed_pair_update_cost = None + ; zkapp_transaction_cost_limit = None + ; max_event_elements = None + ; max_action_elements = None + } + ; genesis = + Some + { k = Some k + ; delta = Some delta + ; slots_per_epoch = Some slots_per_epoch + ; slots_per_sub_window = Some slots_per_sub_window + ; genesis_state_timestamp = + Some Core.Time.(to_string_abs ~zone:Zone.utc @@ now ()) + } + ; proof = Some proof_config (* TODO: prebake ledger and only set hash *) + ; ledger = + Some + { base = + Accounts + (String.Map.data genesis_ledger_accounts |> List.map ~f:fst) + ; add_genesis_winner = None + ; num_accounts = None + ; balances = [] + ; hash = None + ; name = None + } + ; epoch_data = None + } + in + let genesis_constants = + Or_error.ok_exn + (Genesis_ledger_helper.make_genesis_constants ~logger + ~default:Genesis_constants.compiled runtime_config ) + in + let constants : Test_config.constants = + { constraints = constraint_constants; genesis = genesis_constants } + in + (* BLOCK PRODUCER CONFIG *) + let block_producer_config name keypair = + { name; keypair; libp2p_secret = "" } + in + let block_producer_configs = + List.mapi block_producers ~f:(fun n node -> + let _ = + match String.Map.find genesis_ledger_accounts node.account_name with + | Some acct -> + acct + | None -> + let failstring = + Format.sprintf + "Failing because the account key of all initial block \ + producers must be in the genesis ledger. name of Node: \ + %s. name of Account which does not exist: %s" + node.node_name node.account_name + in + failwith failstring + in + block_producer_config node.node_name @@ List.nth_exn network_keypairs n ) + in + let mina_archive_schema = "create_schema.sql" in + let long_commit_id = + let open String in + let id = Mina_version.commit_id in + if prefix id 7 = "[DIRTY]" then suffix id (length id - 7) else id + in + let mina_archive_base_url = + "https://raw.githubusercontent.com/MinaProtocol/mina/" ^ long_commit_id + ^ "/src/app/archive/" + in + let mina_archive_schema_aux_files = + [ mina_archive_base_url ^ "create_schema.sql" + ; mina_archive_base_url ^ "zkapp_tables.sql" + ] + in + let genesis_keypairs = + String.Map.of_alist_exn + (List.map (String.Map.to_alist genesis_ledger_accounts) ~f:(fun element -> + let kp_name, (_, (keypair, _)) = element in + (kp_name, keypair) ) ) + in + let snark_coordinator_config = + match snark_coordinator with + | None -> + None + | Some node -> + let network_kp = + match String.Map.find genesis_keypairs node.account_name with + | Some acct -> + acct + | None -> + let failstring = + Format.sprintf + "Failing because the account key of all initial snark \ + coordinators must be in the genesis ledger. name of Node: \ + %s. name of Account which does not exist: %s" + node.node_name node.account_name + in + failwith failstring + in + Some + { name = node.node_name + ; public_key = + Public_key.Compressed.to_base58_check + (Public_key.compress network_kp.keypair.public_key) + ; worker_nodes = node.worker_nodes + } + in + (* NETWORK CONFIG *) + { debug_arg = debug + ; genesis_keypairs + ; constants + ; config = + { network_id + ; config_dir = cli_inputs.mina_automation_location + ; deploy_graphql_ingress = requires_graphql + ; mina_image = images.mina + ; mina_agent_image = images.user_agent + ; mina_bots_image = images.bots + ; mina_points_image = images.points + ; mina_archive_image = images.archive_node + ; runtime_config = Runtime_config.to_yojson runtime_config + ; block_producer_configs + ; log_precomputed_blocks + ; archive_node_count = List.length archive_nodes + ; mina_archive_schema + ; mina_archive_schema_aux_files + ; snark_coordinator_config + ; snark_worker_fee + } + } From 71e50cf577399c46600c56f1ccac46d2b2eb3cd2 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Mon, 4 Sep 2023 22:59:13 +0000 Subject: [PATCH 53/90] (intg test) Move libp2p keys from test config to topology --- src/lib/integration_test_lib/test_config.ml | 411 ++++++++++++-------- 1 file changed, 244 insertions(+), 167 deletions(-) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 3b22e25cc76..656c8559638 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -78,24 +78,12 @@ module Archive_node = struct end module Block_producer_node = struct - type t = - { node_name : string - ; account_name : string - ; libp2p_keypair : Yojson.Safe.t - ; libp2p_peerid : string - ; docker_image : string - } + type t = { node_name : string; account_name : string; docker_image : string } [@@deriving to_yojson] end module Seed_node = struct - type t = - { node_name : string - ; account_name : string - ; libp2p_keypair : Yojson.Safe.t - ; libp2p_peerid : string - ; docker_image : string - } + type t = { node_name : string; account_name : string; docker_image : string } [@@deriving to_yojson] end @@ -265,123 +253,88 @@ let transactions_needed_for_ledger_proofs ?(num_proofs = 1) config = (blocks_for_first_ledger_proof config * transactions_per_block) + (transactions_per_block * (num_proofs - 1)) -let runtime_config_of_test_config t = - let convert : - Mina_base.Account_timing.t - -> Runtime_config.Accounts.Single.Timed.t option = function - | Untimed -> - None - | Timed timing -> - Some - { initial_minimum_balance = timing.initial_minimum_balance - ; cliff_time = timing.cliff_time - ; cliff_amount = timing.cliff_amount - ; vesting_period = timing.vesting_period - ; vesting_increment = timing.vesting_increment - } - in - let open Yojson.Safe in - let timed_opt_to_yojson : Runtime_config.Accounts.Single.Timed.t option -> t = - function - | None -> - `Null - | Some timed -> - Runtime_config.Accounts.Single.Timed.to_yojson timed - in - let accounts : t = - match to_yojson t |> Util.member "genesis_ledger" with - | `List accounts -> - let accounts = - List.map accounts ~f:(function - | `Assoc account -> - `Assoc - (List.map account ~f:(fun (key, value) -> - ( key - , if String.equal key "timing" then - Mina_base.Account_timing.of_yojson value - |> Result.ok_or_failwith |> convert - |> timed_opt_to_yojson - else value ) ) ) - | _ -> - failwith "Invalid account json" ) - in - `List accounts - | _ -> - failwith "Invalid genesis ledger accounts" +let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : + Topology.t = + let num_bp = List.length t.block_producers in + let num_bp_sc = + num_bp + if Option.is_some t.snark_coordinator then 1 else 0 in - to_string accounts - |> sprintf {| { "accounts": %s } |} - |> from_string |> Runtime_config.Ledger.of_yojson - -let generate_pk_sk () = - let open Signature_lib in - let kp = Keypair.create () in - let sk = kp.private_key |> Private_key.to_base58_check in - let pk = - kp.public_key |> Public_key.compress - |> Public_key.Compressed.to_base58_check + let num_bp_sc_an = num_bp_sc + List.length t.archive_nodes in + let num_bp_sc_an_sn = num_bp_sc_an + List.length t.seed_nodes in + let pk_sk n = + let open Signature_lib in + let sk = List.nth_exn private_keys n in + let pk = + let open Public_key in + Private_key.of_base58_check_exn sk + |> of_private_key_exn |> compress |> Compressed.to_base58_check + in + (pk, sk) in - (pk, sk) - -let topology_of_test_config t : Yojson.Safe.t = - let topology_of_archive { Archive_node.node_name; docker_image; _ } : + let topology_of_block_producer n + { Block_producer_node.node_name; docker_image; _ } : string * Topology.node_info = - let pk, sk = generate_pk_sk () in + let pk, sk = pk_sk n in ( node_name , { pk ; sk - ; role = Archive_node + ; role = Block_producer ; docker_image - ; libp2p_keypair = `Null - ; libp2p_peerid = `Null + ; libp2p_keypair = List.nth_exn libp2p_keypairs n + ; libp2p_peerid = `String List.(nth_exn libp2p_peerids n) } ) in - let topology_of_block_producer - { Block_producer_node.node_name - ; docker_image - ; libp2p_keypair - ; libp2p_peerid - ; _ - } : string * Topology.node_info = - let pk, sk = generate_pk_sk () in + let topology_of_snark_coordinator + { Snark_coordinator_node.node_name; docker_image; _ } : + string * Topology.node_info = + let pk, sk = pk_sk num_bp in ( node_name , { pk ; sk - ; role = Block_producer + ; role = Snark_coordinator ; docker_image - ; libp2p_keypair - ; libp2p_peerid = `String libp2p_peerid + ; libp2p_keypair = `Null + ; libp2p_peerid = `Null } ) in - let topology_of_seed - { Seed_node.node_name; docker_image; libp2p_keypair; libp2p_peerid; _ } : + let snark_coordinator = + match Option.map t.snark_coordinator ~f:topology_of_snark_coordinator with + | None -> + [] + | Some sc -> + [ sc ] + in + let topology_of_archive n { Archive_node.node_name; docker_image; _ } : string * Topology.node_info = - let pk, sk = generate_pk_sk () in + let n = n + num_bp_sc in + let pk, sk = pk_sk n in ( node_name , { pk ; sk - ; role = Seed_node + ; role = Archive_node ; docker_image - ; libp2p_keypair - ; libp2p_peerid = `String libp2p_peerid + ; libp2p_keypair = `Null + ; libp2p_peerid = `Null } ) in - let topology_of_snark_coordinator - { Snark_coordinator_node.node_name; docker_image; _ } : + let topology_of_seed n { Seed_node.node_name; docker_image; _ } : string * Topology.node_info = - let pk, sk = generate_pk_sk () in + let n = n + num_bp_sc_an in + let pk, sk = pk_sk n in ( node_name , { pk ; sk - ; role = Snark_coordinator + ; role = Seed_node ; docker_image - ; libp2p_keypair = `Null - ; libp2p_peerid = `Null + ; libp2p_keypair = List.nth_exn libp2p_keypairs n + ; libp2p_peerid = `String List.(nth_exn libp2p_peerids n) } ) in - let topology_of_snark_worker { Snark_worker_node.node_name; docker_image; _ } - : string * Topology.node_info = - let pk, sk = generate_pk_sk () in + let topology_of_snark_worker n + { Snark_worker_node.node_name; docker_image; _ } : + string * Topology.node_info = + let n = n + num_bp_sc_an_sn in + let pk, sk = pk_sk n in ( node_name , { pk ; sk @@ -391,73 +344,37 @@ let topology_of_test_config t : Yojson.Safe.t = ; libp2p_peerid = `Null } ) in - let snark_coordinator = - match Option.map t.snark_coordinator ~f:topology_of_snark_coordinator with - | None -> - [] - | Some sc -> - [ sc ] - in - let open List in snark_coordinator - @ map t.archive_nodes ~f:topology_of_archive - @ map t.block_producers ~f:topology_of_block_producer - @ map t.seed_nodes ~f:topology_of_seed - @ map t.snark_workers ~f:topology_of_snark_worker - |> Topology.to_yojson - -let test_account ?(pk = generate_pk_sk () |> fst) - ?(timing = Mina_base.Account.Timing.Untimed) account_name balance : - Test_Account.t = + @ List.mapi t.archive_nodes ~f:topology_of_archive + @ List.mapi t.block_producers ~f:topology_of_block_producer + @ List.mapi t.seed_nodes ~f:topology_of_seed + @ List.mapi t.snark_workers ~f:topology_of_snark_worker + +let test_account ?(pk = "") ?(timing = Mina_base.Account.Timing.Untimed) + account_name balance : Test_Account.t = { account_name; balance; timing; pk } -let bp ?(libp2p_keypair = `Null) ?(libp2p_peerid = "") node_name - ?(account_name = node_name ^ "-key") docker_image = - { Block_producer_node.node_name - ; account_name - ; libp2p_keypair - ; libp2p_peerid - ; docker_image - } +let bp node_name ?(account_name = node_name ^ "-key") docker_image = + { Block_producer_node.node_name; account_name; docker_image } module Unit_tests = struct let test_config = { default with genesis_ledger = - [ test_account "receiver-key" "9999999" - ; test_account "empty-bp-key" "0" + [ test_account "receiver" "9999999" + ; test_account "empty-bp" "0" ; test_account "snark-node-key" "0" ] @ List.init 2 ~f:(fun i -> - let i_str = Int.to_string i in - test_account (sprintf "sender-account-%s" i_str) "10000" ) + test_account (sprintf "sender-account-%d" i) "10000" ) ; block_producers = [ { node_name = "receiver" ; account_name = "receiver-key" ; docker_image = "bp-image-0" - ; libp2p_keypair = - Yojson.Safe.from_string - {json| - { - "box_primitive": "xsalsa20poly1305", - "pw_primitive": "argon2i", - "nonce": "8tXZ3yVev59WGBFDhVkXYsqAkAy1oa4a9GkAgke", - "pwsalt": "AbLiWBzfVzdPK2YYRhhUxvQv7W8C", - "pwdiff": [ - 134217728, - 6 - ], - "ciphertext": "8uu1u388h2YJDfQGdAPurUrTgWGiUuRfz2t6TrRPEaqQoPVMHBvZFSC7BMZwCqoWk1PNAQVEPuBLQAhcBtdktmrjBnjhr2P38UiDzSa3gX7uawToQBV1WJh2gTNwPBYh2YRnRZNUY3ThaqqJmx7t7rrApYSqbSoAsLjydGSRYhxdrPtHeokmtcbtzFJ65Q5SWL6vAdhJqCqMcGXUhLDMpCsBwreqkFxn3QmQ9UinG1HTKGNV2FyrXDKUxseboBB3EEmq1Xvmt7ZaPropVcWeL9doWwYVW3xjiMXwR" - } - |json} - ; libp2p_peerid = - "12D3KooWAZBFbTQWmegLFt8YSExEMTwJEiufRf4skYdZtaGxEXXo" } - ; { node_name = "empty_node" + ; { node_name = "empty-node" ; account_name = "empty-bp-key" ; docker_image = "bp-image-1" - ; libp2p_keypair = `Null - ; libp2p_peerid = "" } ] ; snark_coordinator = @@ -467,6 +384,12 @@ module Unit_tests = struct ; docker_image = "snark-coordinator-image" ; worker_nodes = 4 } + ; seed_nodes = + [ { node_name = "seed-node-0" + ; account_name = "seed-node-0-key" + ; docker_image = "seed-image-0" + } + ] ; txpool_max_size = 10_000_000 ; snark_worker_fee = "0.0001" ; proof_config = @@ -477,23 +400,177 @@ module Unit_tests = struct } } - open Yojson.Safe - - let%test_unit "runtime_config_of_test_config" = - let config_json = - runtime_config_of_test_config test_config - |> Result.ok_or_failwith |> Runtime_config.Ledger.to_yojson - in - let () = - print_endline "=== Runtime config ===" ; - pretty_to_string config_json |> print_endline - in - ignore (config_json : t) + let private_keys = + [ "EKEGuARLLw4uxXQ1WLL75HQhZpYSMqiXSc5Y5rbNFC7JvwWT6vgr" + ; "EKF7fXWzfSCmBUXU6nDXMvHuVFQWae4rRwvjALybczfe3rVk3893" + ; "EKEq5fbQzUyvuzPLb1mdd9McpAbXX8cuyJVHSYaCurnyQS6xYFvx" + ; "EKDrCY8JTBqscUz992LjZs8Jbf7LRVmH5XzCPeoGAveMvo5ryoZJ" + ; "EKDp3HVFL6PfDPuXeLnLzBq1cEbv1j18jWynPuSwFBgRHard1LFP" + ; "EKF8miHH6qJY5YTncLNFJZzpaBm3GCkH9VZMNS15CxdgLPp8Y19D" + ; "EKEp3pmqba7oBRnetJvmyFev9gEU6bGe5ELBDcBgYYP9gXrZadJ3" + ; "EKDkDNB3cDhfjWkfH1t5WnkiH92NQmdYpnNE3HybTzcm5ncQsW6o" + ; "EKDmK3a1bnaNu7g1dJM2X2coLXDgWAfLmG5kYmyxHHTDpd4FqXFE" + ] + + let libp2p_keypairs = + List.map ~f:Yojson.Safe.from_string + [ {json| + { + "box_primitive": "xsalsa20poly1305", + "pw_primitive": "argon2i", + "nonce": "7NJv3URmZRNFvGp3wjbfKNtGsJRLXoqZewJenM9", + "pwsalt": "8nLZQAJZBaFgKz9kNjs6kj6HAPnJ", + "pwdiff": [ + 134217728, + 6 + ], + "ciphertext": "8SpinAQfG7vhvpceA4bPoyU3v3CoiJcNA75JZr5EWb9631biapTLX2AUryDyMtVYYtV26ahf8NWEYJsPbeLmNtszumPBbzozGcasQTJGCVa7EacEoCTGCLVpgVZphRDkPHFbpgGTUpFTgaxa4NpUpPtrkUFTpDtEgY8dtpC27RHyj83jS8GCrfznMqYKWSiGCMfBfX5K3KH2Z1PQZB6iL28cqbYx3YnAf4wQNFyemWfH7MzBzhRLXX6CHqWd5m71fUUuv5F2C9TLqeqbD75ZKbikrhNY43UDPEsgC" + } + |json} + ; {json| + { + "box_primitive": "xsalsa20poly1305", + "pw_primitive": "argon2i", + "nonce": "6ogEKiFNXwko1WELXrLsjNovy7WWhm5qkdX7dDR", + "pwsalt": "BLeWoEHTgFags57xEgbQvT5FYprq", + "pwdiff": [ + 134217728, + 6 + ], + "ciphertext": "8NCrBdZbtQ9JTwTj9a1GWbQihjMXUZnotuctEyEq5xv6aN3Pecf2o54skHB6ezqeuhD5xnQSTjvG7hh15UKo6QJFongyVZsA5DxvLYcgXWbT26j8UzihMByuY1CzsF3zgwpkYLha63VwJT4irqM9vkD1YeaGC9wjLrXdiVWML9bXvMpE2g7YgSETdLsPCGBbAmg1FSg26gkWoVEnkHPaajagqtMuS9N2nJxjLk3w55V9PbkzvzcBW2134mro6MJgsp9BYdLkDjZMyRw7FtHcWm6HvTXML2rw78QXz" + } + |json} + ; {json| + { + "box_primitive": "xsalsa20poly1305", + "pw_primitive": "argon2i", + "nonce": "8rQDwmqSnZMiaYPXHhVpimLU716M4pGnebMeE7W", + "pwsalt": "BbWDiSvA7wPkzsGLMepeQVV5o7HJ", + "pwdiff": [ + 134217728, + 6 + ], + "ciphertext": "8VZJzft4R7EH7Fawgaxb1hERPztsaKURHtHkFTH1L2vmkJtPoUGQecvFCePXaAnK5LzLmBkczSNAxACmmt2ojjGXnyCVTFgjiaUKHfQ244ur2jBEQp2kkz6ZTgZDZZ2UH4WcsamPJyjmnBCNSP8B7C3w2pgarpozT49w9i87KuYJWJywRKytoxHM747ydSNuDo3ooQrNewhgbJV6S71ikFYdMQc9ejKZDMv6n4zPj8c913TYj1JQSgKA14gegaGAZZtdRXwgJCKxQW4WVyeLRzNMeLkEyn9kND85a" + } + |json} + ; {json| + { + "box_primitive": "xsalsa20poly1305", + "pw_primitive": "argon2i", + "nonce": "6wwcrYF73ZdxRPAM4PDRhGY8QMRYdgEJDQ5jVVu", + "pwsalt": "A3xzWd5eUT9xrQkGF2xrKdYbm2d9", + "pwdiff": [ + 134217728, + 6 + ], + "ciphertext": "7znhmqfvuUGPDE4js899VGVXv4tHd6LSXZCmnfCj14642KjzA4BcLY9maJiVaq5YCpUhPLVRjMPWDnovoAb5BKLmcbVywvPFSDREbhKdBh24u3z3wYzphAr9vZkPJiDNRXhFZ7b9JPoBchdJT4RZxKsHBbbpBmhBDuqM395YaPF1d2jPg5P5quivK98BLvX26SiJ8KAeswKk5YsoC3irA4PbZ5E7AQ5TfRCZj7k8c2g5f6awn18fcg4xt6hUwZ8o3usuqNX62HXDhETjGfXfYUvsam25522vj7Pzu" + } + |json} + ; {json| + { + "box_primitive": "xsalsa20poly1305", + "pw_primitive": "argon2i", + "nonce": "7r3DGH71WaBsGA2LXV1RWNys5Vj3rGsLxiAeUxi", + "pwsalt": "8NQcBj9grs7Gx8L77C16XXcjTvPD", + "pwdiff": [ + 134217728, + 6 + ], + "ciphertext": "934bNqreFhzuC36gpi8j5tAiL38huhwkdTgPR3CfMy7GPS1611ryVHGtoEayS1h7m77VdHMTvP5kgX5ZpExk2qAP4Rd1Zac2iVB9ZGUDmnSrR4Lp7fthYPmMPEzKtyXBMszjzFcLoiTiyYc18a8ZgwG14GvMvED8dHQz1cdHWzJBTptXQtMzd12J7NSsnBcqP8ZKrF68G3LWE6ehck6Q1wC11jAwwzq4tiWLgQpf4z2hU7tbdMJjMFbP3pNUPX2BeV8eUcsn8Jd16jwyqdWzSXDj1Kj5rAPo3V7eA" + } + |json} + ; {json| + { + "box_primitive": "xsalsa20poly1305", + "pw_primitive": "argon2i", + "nonce": "7CWMdMioqAXuCbXAGCySnNyvHNv3biiwtephci8", + "pwsalt": "9bjDaPFvjDQWP9nycZWG4wWTsutJ", + "pwdiff": [ + 134217728, + 6 + ], + "ciphertext": "8XBiDBr2dMwFTwvnHE4RsQDYVL5ycHAuZATskMURr3wmAWhpqfMJUnhQmVk6fJgiVzcfqks6ha2qhNCcv6BTxp5MCgKSHT5v4q4Ei89HPRUxthGrvWzf9xwzyCn1gcZyK25v5VhUFFwx2U7pG89iVaGHrwMMDH5onGTMUWJ5LVwRZhbp7mfraTYPZPanhBQz3Dqw9z21SYbvaS23q59D9swrYVB9Ncu7kjh7exteqXCEwmPgjDJiTSv78VCNgWAxB1tRFejqwMUuHJQVZTHnUcAtirGYz3w4iagGQ" + } + |json} + ; {json| + { + "box_primitive": "xsalsa20poly1305", + "pw_primitive": "argon2i", + "nonce": "8dk4cDaooY3yc3rf149wucqeEbb89FmoCMnXu97", + "pwsalt": "Ap3ew2UusjQa973JYv7LJYUtugde", + "pwdiff": [ + 134217728, + 6 + ], + "ciphertext": "8KVBFy4cc4E4JCgvGQbH6k32nxzN4JD12YMJmU8UiGg7m6VVPrYxA1HNnxtzNabkvRZQA9s2fhiGxDqyyZuxch27KFKBUYwtNDsFquCx2uQPM1aTUMPDmDnduNVy4ZjaSPBVwqGcTjLqVTjGzsMj3oW6ASMoBJB7E94UekNuLQwMz5iyawzQPshBKdcCauL97mKyzPM2pyWeYCxBXAuCLopsqq1CtsZCdiGXH1Rvx6uuLJDQwL2N5vHeZFpULTuo1kYqry8iVra3Yfic83vHcsaY2a9MPm1pKoyAM" + } + |json} + ; {json| + { + "box_primitive": "xsalsa20poly1305", + "pw_primitive": "argon2i", + "nonce": "7LrW1q7Lvg5vQD5vgL1eKnAT157WdnJHnS5u5Uv", + "pwsalt": "AnZ678uLBtfzwryiZrzHFSc2CB3d", + "pwdiff": [ + 134217728, + 6 + ], + "ciphertext": "8xtTBGmEZWKa3PQtVF47BWJk2u6saHpZ7mo4e3b1nLJNUmLZoUq5cg8MyDWntsqkTQiiixWLWrnjV192e4qy1ULF4CywYWRz5xGHGeFVUVj88gJvLQ9gqQKjbA39mTB29jjVhyyUbmoMi7FALLYx6p9h7rbAWTam3a3QCmg4nUksqKSDcxyihP1QFtfHZb7DxcBoYZUb5LDA5Xp7bu92NkDZmmn4VHtRDmtxv7VoqATmeL3NhypTbgqwfCAj2PxB28wL8STLCwtB8uqxqxRTA8gRJ7gpMEM1Js4fm" + } + |json} + ; {json| + { + "box_primitive": "xsalsa20poly1305", + "pw_primitive": "argon2i", + "nonce": "7fKJMu1E7Qny5ma4EuRiLFXQpoTMv6b8GU9E3jt", + "pwsalt": "8z7PecFQ7CezTQjmbdm4NG6sFTSy", + "pwdiff": [ + 134217728, + 6 + ], + "ciphertext": "7GBDTTuhwbWME11jYZkjsCguJgfzR6NcejpJbvsBiGiyWbzvaQrEfTmSd4KaVZkp6UQZ8U3QK54WSW6x1mFPoJmDT2xGHcekTuBccWe5KSU1MEtxhtmbRyWkpT14T22Rrt4b5f4XimySxFAhnex6vwwZgKGANAmHSvNp9mEqaVXcPS2VuniyTsN72wMjg8uHM4jo3YALDKWXb8t4uhZTv7QQw7N51gfYEBkcZxfXAWdBK75cMGSK2VgXgxFrCKSE81D76xBBqdn2nPLZQYW2yqL9MrjaMoaj7bP9G" + } + |json} + ] + + let libp2p_peerids = + [ "12D3KooWJE7cTbqhsY1qpoXeXEFhEHu14FVJEJ69brG6mkbvtVuP" + ; "12D3KooWBEwyZgnSojuUSeU8oSwmAdPYoA2Pn8uqMPCdWHNTjx8h" + ; "12D3KooWEMRzv3RKHsMgFqmFEZAQ12c4QhRsg4J8uYye5wWRzQA6" + ; "12D3KooWE9Jw7s97bEnDJCen4v2sZGREH6jQfKso7Qp3kF7WxU75" + ; "12D3KooWP5ae8JYBVdX3KnsnfW8ohn3M6FCBEyitSrC4h9Vy8cvQ" + ; "12D3KooWL85m2z7hrvg6NMtE9ibE44mBDjnuqYSWwhwJMPQe6gtD" + ; "12D3KooWLTdMy4Jt2zAT6JsgnmVznB2zvYNr2eybVGumdhuhgN5U" + ; "12D3KooWECCM7imLRwh7RExK64Vwf14edwctq7U8ubZ2muZUuXgE" + ; "12D3KooWJnNwLdE82a2WcSvkpLMHicL3onayZJaSEk1M1uPkmbH7" + ] let%test_unit "topology_of_test_config" = - let () = - print_endline "\n=== Topology ===" ; - topology_of_test_config test_config |> pretty_to_string |> print_endline + let open Yojson.Safe in + let topology = + topology_of_test_config test_config private_keys libp2p_keypairs + libp2p_peerids + |> Topology.to_yojson + in + print_endline "=== Topology ===" ; + topology |> pretty_to_string |> print_endline ; + (* only block producers and seed nodes get libp2p keypairs *) + let get_libp2p_keypair node_name = + Util.(member node_name topology |> member "libp2p_keypair") in - ignore (topology_of_test_config test_config : t) + assert ( + List.for_all test_config.block_producers ~f:(fun bp -> + not @@ equal `Null @@ get_libp2p_keypair bp.node_name ) ) ; + assert ( + List.for_all test_config.seed_nodes ~f:(fun sn -> + not @@ equal `Null @@ get_libp2p_keypair sn.node_name ) ) ; + assert ( + List.for_all test_config.archive_nodes ~f:(fun an -> + equal `Null @@ get_libp2p_keypair an.node_name ) ) ; + assert ( + List.for_all test_config.snark_workers ~f:(fun sw -> + equal `Null @@ get_libp2p_keypair sw.node_name ) ) ; + assert ( + Option.for_all test_config.snark_coordinator ~f:(fun sc -> + equal `Null @@ get_libp2p_keypair sc.node_name ) ) end From 2d54d5fc773b82f97da94fd0e88e57baabbb750e Mon Sep 17 00:00:00 2001 From: Quantifier Date: Mon, 4 Sep 2023 23:20:11 +0000 Subject: [PATCH 54/90] (intg test) Handle all topology info in network config --- src/app/test_executive/mock.ml | 20 ++++- .../mina_automation.ml | 60 +++++++-------- .../network_config.ml | 77 ++++++++++++------- .../unit_tests.ml | 5 +- 4 files changed, 104 insertions(+), 58 deletions(-) diff --git a/src/app/test_executive/mock.ml b/src/app/test_executive/mock.ml index 54125bece3b..e5b5849e6c7 100644 --- a/src/app/test_executive/mock.ml +++ b/src/app/test_executive/mock.ml @@ -22,7 +22,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let i_str = Int.to_string i in test_account ("sender-account-" ^ i_str) "10000" ) ; block_producers = - [ bp "receiver" !Network.mina_image + [ bp "receiver" "another-docker-image" ; bp "empty_node-1" ~account_name:"empty-bp-key" !Network.mina_image ; bp "empty_node-2" ~account_name:"empty-bp-key" !Network.mina_image ; bp "empty_node-3" ~account_name:"empty-bp-key" !Network.mina_image @@ -44,6 +44,24 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ; transaction_capacity = Some Runtime_config.Proof_keys.Transaction_capacity.small } + ; seed_nodes = + [ { node_name = "seed-0" + ; account_name = "seed-0-key" + ; docker_image = "seed-docker-image" + } + ] + ; snark_workers = + [ { node_name = "snark-0" + ; account_name = "snark-0-key" + ; docker_image = "snark-docker-image" + } + ] + ; archive_nodes = + [ { node_name = "archive-0" + ; account_name = "archive-0-key" + ; docker_image = "archive-docker-image" + } + ] } let run _network _t = Malleable_error.return () diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index e9ef61e9b72..96e0f53733c 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -25,40 +25,42 @@ module Network_manager = struct let run_cmd_or_hard_error t prog args = Util.run_cmd_or_hard_error t.testnet_dir prog args - let create ~logger (network_config : Network_config.t) - (test_config : Test_config.t) = + let create ~logger (network_config : Network_config.t) _test_config = let open Malleable_error.Let_syntax in + let testnet_dir = + network_config.config.config_dir ^/ "/testnets" + ^/ network_config.config.network_id + in let%bind () = - if network_config.debug_arg then + if Stdlib.Sys.file_exists testnet_dir then Deferred.bind ~f:Malleable_error.return - (Util.prompt_continue - "Existing namespace of same name detected, pausing startup. Enter \ - [y/Y] to continue on and remove existing namespace, start clean, \ - and run the test; press Ctrl-C to quit out: " ) + (Deferred.map + (Util.prompt_continue + "Existing namespace of same name detected, pausing startup. \ + Enter [y/Y] to continue on and remove existing namespace, \ + start clean, and run the test; press Ctrl-C to quit out: " ) + ~f:(fun () -> + let rec rmrf path = + let open Stdlib in + match Sys.is_directory path with + | true -> + Sys.readdir path + |> Array.iter (fun name -> + rmrf (Filename.concat path name) ) ; + Core.Unix.rmdir path + | false -> + Sys.remove path + in + [%log info] "Deleting old testnet dir %s" testnet_dir ; + rmrf testnet_dir ) ) else Malleable_error.return () in (* TODO: prebuild genesis proof and ledger and cache for future use *) let testnet_log_filter = Network_config.testnet_log_filter network_config in (* we currently only deploy 1 seed and coordinator per deploy (will be configurable later) *) (* seed node keyname and workload name hardcoded as "seed" *) - let testnet_dir = - network_config.config.config_dir ^/ "/testnets" - ^/ network_config.config.network_id - in - let open Deferred.Let_syntax in - [%log info] "Making testnet dir %s" testnet_dir ; - let rec rmrf path = - let open Stdlib in - match Sys.is_directory path with - | true -> - Sys.readdir path - |> Array.iter (fun name -> rmrf (Filename.concat path name)) ; - Core.Unix.rmdir path - | false -> - Sys.remove path - in - if Stdlib.Sys.file_exists testnet_dir then rmrf testnet_dir ; - let () = mkdir_p testnet_dir in + [%log info] "Making new testnet dir %s" testnet_dir ; + mkdir_p testnet_dir ; let network_config_filename = testnet_dir ^/ "network_config.json" in let runtime_config_filename = testnet_dir ^/ "runtime_config.json" in let topology_filename = testnet_dir ^/ "topology.json" in @@ -82,15 +84,13 @@ module Network_manager = struct [%log info] "Writing runtime configuration to %s" runtime_config_filename ; Out_channel.with_file ~fail_if_exists:true runtime_config_filename ~f:(fun ch -> - Test_config.runtime_config_of_test_config test_config - |> Result.ok_or_failwith |> Runtime_config.Ledger.to_yojson - |> Yojson.Safe.to_channel ch ) ; + network_config.config.runtime_config |> Yojson.Safe.to_channel ch ) ; [%log info] "Writing topology to %s" topology_filename ; Out_channel.with_file ~fail_if_exists:true topology_filename ~f:(fun ch -> - Test_config.topology_of_test_config test_config - |> Yojson.Safe.to_channel ch ) ; + network_config.config.topology |> Yojson.Safe.to_channel ch ) ; [%log info] "Writing out the genesis keys to testnet dir %s" testnet_dir ; let kps_base_path = testnet_dir ^ "/genesis_keys" in + let open Deferred.Let_syntax in let%bind () = Unix.mkdir kps_base_path in let%bind () = Core.String.Map.iter network_config.genesis_keypairs ~f:(fun kp -> diff --git a/src/lib/integration_test_abstract_engine/network_config.ml b/src/lib/integration_test_abstract_engine/network_config.ml index 1890c1a098d..38cf4f44b7e 100644 --- a/src/lib/integration_test_abstract_engine/network_config.ml +++ b/src/lib/integration_test_abstract_engine/network_config.ml @@ -10,7 +10,12 @@ module Cli_inputs = Cli_inputs let keypairs_path = ref "" type block_producer_config = - { name : string; keypair : Network_keypair.t; libp2p_secret : string } + { name : string + ; keypair : Network_keypair.t + ; libp2p_secret : string + ; libp2p_keypair : Yojson.Safe.t + ; libp2p_peerid : string + } [@@deriving to_yojson] type snark_coordinator_config = @@ -35,6 +40,8 @@ type config = ; mina_archive_schema_aux_files : string list ; snark_coordinator_config : snark_coordinator_config option ; snark_worker_fee : string + ; topology : Yojson.Safe.t + [@to_yojson fun j -> `String (Yojson.Safe.to_string j)] } [@@deriving to_yojson] @@ -71,7 +78,8 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ; slots_per_epoch ; slots_per_sub_window ; txpool_max_size - ; _ + ; seed_nodes + ; snark_workers } = test_config in @@ -102,8 +110,14 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) (* ACCOUNTS AND KEYPAIRS *) let before = Time.now () in let num_keypairs = List.length genesis_ledger in - let network_keypairs, private_keys, _libp2p_keypairs, _libp2p_peerids = - Util.pull_keypairs !keypairs_path (List.length genesis_ledger) + let network_keypairs, private_keys, libp2p_keypairs, libp2p_peerids = + let max_num_nodes = + List.length archive_nodes + + List.length block_producers + + List.length seed_nodes + List.length snark_workers + 1 + in + Util.pull_keypairs !keypairs_path + (List.length genesis_ledger + max_num_nodes) in [%log trace] "Pulled %d keypairs from %s in %s" num_keypairs !keypairs_path Time.(abs_diff before @@ now () |> Span.to_string) ; @@ -154,13 +168,18 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) tl in let genesis_ledger_accounts = + let num_genesis_accounts = List.length genesis_ledger in let network_keypairs = List.mapi network_keypairs ~f:(fun n kp -> - { kp with - keypair_name = (List.nth_exn genesis_ledger n).account_name - } ) + if n < num_genesis_accounts then + { kp with + keypair_name = (List.nth_exn genesis_ledger n).account_name + } + else kp ) + in + let keypairs = + List.(take (zip_exn network_keypairs private_keys) num_genesis_accounts) in - let keypairs = List.zip_exn network_keypairs private_keys in add_accounts labeled_accounts (List.zip_exn genesis_ledger keypairs) in (* DAEMON CONFIG *) @@ -213,32 +232,33 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) { constraints = constraint_constants; genesis = genesis_constants } in (* BLOCK PRODUCER CONFIG *) - let block_producer_config name keypair = - { name; keypair; libp2p_secret = "" } + let block_producer_config n name = + { name + ; keypair = List.nth_exn network_keypairs n + ; libp2p_secret = "" + ; libp2p_keypair = List.nth_exn libp2p_keypairs n + ; libp2p_peerid = List.nth_exn libp2p_peerids n + } in let block_producer_configs = List.mapi block_producers ~f:(fun n node -> - let _ = - match String.Map.find genesis_ledger_accounts node.account_name with - | Some acct -> - acct - | None -> - let failstring = - Format.sprintf - "Failing because the account key of all initial block \ - producers must be in the genesis ledger. name of Node: \ - %s. name of Account which does not exist: %s" - node.node_name node.account_name - in - failwith failstring - in - block_producer_config node.node_name @@ List.nth_exn network_keypairs n ) + if + Option.is_none + @@ String.Map.find genesis_ledger_accounts node.account_name + then + failwith + @@ Format.sprintf + "Failing because the account key of all initial block producers \ + must be in the genesis ledger. name of Node: %s. name of \ + Account which does not exist: %s" + node.node_name node.account_name ; + block_producer_config n node.node_name ) in let mina_archive_schema = "create_schema.sql" in let long_commit_id = let open String in let id = Mina_version.commit_id in - if prefix id 7 = "[DIRTY]" then suffix id (length id - 7) else id + if prefix id 7 = "[DIRTY]" then drop_prefix id 7 else id in let mina_archive_base_url = "https://raw.githubusercontent.com/MinaProtocol/mina/" ^ long_commit_id @@ -303,5 +323,10 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ; mina_archive_schema_aux_files ; snark_coordinator_config ; snark_worker_fee + ; topology = + Test_config.( + topology_of_test_config test_config private_keys libp2p_keypairs + libp2p_peerids + |> Topology.to_yojson) } } diff --git a/src/lib/integration_test_abstract_engine/unit_tests.ml b/src/lib/integration_test_abstract_engine/unit_tests.ml index 954111f00f9..fee35e30e0d 100644 --- a/src/lib/integration_test_abstract_engine/unit_tests.ml +++ b/src/lib/integration_test_abstract_engine/unit_tests.ml @@ -89,6 +89,7 @@ module Test_values = struct let mina_archive_schema_aux_files = [] in let snark_coordinator_config = None in let snark_worker_fee = "0.01" in + let topology = `Assoc [] in { network_id ; config_dir ; deploy_graphql_ingress @@ -105,6 +106,7 @@ module Test_values = struct ; mina_archive_schema_aux_files ; snark_coordinator_config ; snark_worker_fee + ; topology } end @@ -319,7 +321,8 @@ module Request_tests = struct "mina_archive_schema": "schema", "mina_archive_schema_aux_files": [], "snark_coordinator_config": null, - "snark_worker_fee": "0.01" + "snark_worker_fee": "0.01", + "topology": "{}" } } |} From ac7666243adb221b20080844ceb23acaf0ccbcd0 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Mon, 4 Sep 2023 23:24:28 +0000 Subject: [PATCH 55/90] (test exec) Clean up file paths --- src/app/test_executive/test_executive.ml | 4 ++-- .../abstract_network.ml | 4 +--- src/lib/integration_test_lib/util.ml | 12 +++++------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index 8bad3986c6b..0043e188c7f 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -76,7 +76,7 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : if not ( Array.exists keypairs_ls ~f:(String.equal "network-keypairs") - && (Stdlib.Sys.is_directory @@ keypairs_path ^ "/network-keypairs") ) + && (Stdlib.Sys.is_directory @@ keypairs_path ^/ "network-keypairs") ) then ( [%log fatal] "No network-keypairs directory present in %s \n\ @@ -88,7 +88,7 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : if not ( Array.exists keypairs_ls ~f:(String.equal "libp2p-keypairs") - && (Stdlib.Sys.is_directory @@ keypairs_path ^ "/libp2p-keypairs") ) + && (Stdlib.Sys.is_directory @@ keypairs_path ^/ "libp2p-keypairs") ) then ( [%log fatal] "No libp2p-keypairs directory present in %s \n\ diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index 34cb1ed3b45..ba30500293e 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -1290,6 +1290,4 @@ let all_node_id t = let pods = all_pods t |> Core.Map.to_alist in List.fold pods ~init:[] ~f:(fun acc (_, node) -> node.node_id :: acc) -let initialize_infra ~logger network = - [%log trace] "initialize_infra: %s" network.network_id ; - Malleable_error.return () +let[@warning "-27"] initialize_infra ~logger network = Malleable_error.return () diff --git a/src/lib/integration_test_lib/util.ml b/src/lib/integration_test_lib/util.ml index 10b226ce591..f9f33990615 100644 --- a/src/lib/integration_test_lib/util.ml +++ b/src/lib/integration_test_lib/util.ml @@ -29,13 +29,9 @@ let pull_keypairs path num_keypairs = Quickcheck.Generator.(of_list int_list |> list_with_length num_keypairs) |> Quickcheck.random_value in - let normalize_path path = - if String.(suffix path 1 = "/") then String.drop_suffix path 1 else path - in (* network keypairs + private keys *) - let keypairs_path = normalize_path path in let base_filename n = - sprintf "%s/network-keypairs/sender-account-%d.json" keypairs_path n + path ^/ sprintf "network-keypairs/sender-account-%d.json" n in let read_sk n = In_channel.read_all (base_filename n ^ ".sk") in let read_keypair n = @@ -52,10 +48,12 @@ let pull_keypairs path num_keypairs = in (* libp2p keypairs *) let libp2p_base_filename n = - sprintf "%s/libp2p-keypairs/libp2p-keypair-%d.json" keypairs_path n + path ^/ sprintf "libp2p-keypairs/libp2p-keypair-%d.json" n in let read_peerid n = - In_channel.read_all (libp2p_base_filename n ^ ".peerid") + let raw = In_channel.read_all (libp2p_base_filename n ^ ".peerid") in + let open String in + if suffix raw 1 = "\n" then drop_suffix raw 1 else raw in let read_libp2p n = Yojson.Safe.from_file (libp2p_base_filename n) in ( List.map random_nums ~f:read_keypair From 8485910284ed86e78415ffbe0b6c90da8600ce0d Mon Sep 17 00:00:00 2001 From: Quantifier Date: Mon, 4 Sep 2023 23:55:09 +0000 Subject: [PATCH 56/90] (intg test) Randomize keypairs used in tests --- src/lib/integration_test_lib/dune | 1 + src/lib/integration_test_lib/util.ml | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/lib/integration_test_lib/dune b/src/lib/integration_test_lib/dune index d6fa42f98cd..bbe92b5dba8 100644 --- a/src/lib/integration_test_lib/dune +++ b/src/lib/integration_test_lib/dune @@ -28,6 +28,7 @@ stdio extlib sexp_diff_kernel + splittable_random ;; local libraries key_gen visualization diff --git a/src/lib/integration_test_lib/util.ml b/src/lib/integration_test_lib/util.ml index f9f33990615..5f9f2225657 100644 --- a/src/lib/integration_test_lib/util.ml +++ b/src/lib/integration_test_lib/util.ml @@ -24,10 +24,14 @@ let drop_outer_quotes s = s let pull_keypairs path num_keypairs = - let int_list = List.range ~stop:`inclusive 1 10_000 in let random_nums = - Quickcheck.Generator.(of_list int_list |> list_with_length num_keypairs) - |> Quickcheck.random_value + let random = + Random.State.make_self_init () |> Splittable_random.State.create + in + let open Quickcheck.Generator in + of_list (List.range ~stop:`inclusive 1 10_000) + |> list_with_length num_keypairs + |> generate ~size:num_keypairs ~random in (* network keypairs + private keys *) let base_filename n = From 4a1974c4044ffc9cef8d6080c6fd9af8a2886de3 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 5 Sep 2023 00:28:39 +0000 Subject: [PATCH 57/90] (intg test) Rename bp network keypairs --- src/lib/integration_test_abstract_engine/network_config.ml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/integration_test_abstract_engine/network_config.ml b/src/lib/integration_test_abstract_engine/network_config.ml index 38cf4f44b7e..9c52e29da59 100644 --- a/src/lib/integration_test_abstract_engine/network_config.ml +++ b/src/lib/integration_test_abstract_engine/network_config.ml @@ -233,8 +233,9 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) in (* BLOCK PRODUCER CONFIG *) let block_producer_config n name = + let keypair = List.nth_exn network_keypairs n in { name - ; keypair = List.nth_exn network_keypairs n + ; keypair = { keypair with keypair_name = name } ; libp2p_secret = "" ; libp2p_keypair = List.nth_exn libp2p_keypairs n ; libp2p_peerid = List.nth_exn libp2p_peerids n From acedf4c0d9f6824cae3b3adce203a7ba070d133b Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 5 Sep 2023 23:48:16 +0000 Subject: [PATCH 58/90] (intg test) Add seed node configs to network config --- .../network_config.ml | 63 ++++++++++++++----- .../unit_tests.ml | 3 + 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/network_config.ml b/src/lib/integration_test_abstract_engine/network_config.ml index 9c52e29da59..ad53f52c2cc 100644 --- a/src/lib/integration_test_abstract_engine/network_config.ml +++ b/src/lib/integration_test_abstract_engine/network_config.ml @@ -12,7 +12,15 @@ let keypairs_path = ref "" type block_producer_config = { name : string ; keypair : Network_keypair.t - ; libp2p_secret : string + ; libp2p_pass : string + ; libp2p_keypair : Yojson.Safe.t + ; libp2p_peerid : string + } +[@@deriving to_yojson] + +type seed_node_config = + { name : string + ; libp2p_pass : string ; libp2p_keypair : Yojson.Safe.t ; libp2p_peerid : string } @@ -34,6 +42,7 @@ type config = ; runtime_config : Yojson.Safe.t [@to_yojson fun j -> `String (Yojson.Safe.to_string j)] ; block_producer_configs : block_producer_config list + ; seed_node_configs : seed_node_config list ; log_precomputed_blocks : bool ; archive_node_count : int ; mina_archive_schema : string @@ -121,6 +130,10 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) in [%log trace] "Pulled %d keypairs from %s in %s" num_keypairs !keypairs_path Time.(abs_diff before @@ now () |> Span.to_string) ; + let topology = + Test_config.topology_of_test_config test_config private_keys libp2p_keypairs + libp2p_peerids + in let labeled_accounts : (Runtime_config.Accounts.single * (Network_keypair.t * Private_key.t)) String.Map.t = @@ -232,17 +245,25 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) { constraints = constraint_constants; genesis = genesis_constants } in (* BLOCK PRODUCER CONFIG *) - let block_producer_config n name = - let keypair = List.nth_exn network_keypairs n in - { name - ; keypair = { keypair with keypair_name = name } - ; libp2p_secret = "" - ; libp2p_keypair = List.nth_exn libp2p_keypairs n - ; libp2p_peerid = List.nth_exn libp2p_peerids n + let block_producer_config (node : Test_config.Block_producer_node.t) = + let _, (keypair, _) = + Core.String.Map.find_exn genesis_ledger_accounts node.account_name + in + let node_info = + List.find_exn topology ~f:(fun (name, _) -> + String.equal name node.node_name ) + |> snd + in + { name = node.node_name + ; keypair = { keypair with keypair_name = node.node_name } + ; libp2p_pass = "naughty blue worm" + ; libp2p_keypair = node_info.libp2p_keypair + ; libp2p_peerid = + Yojson.Safe.to_string node_info.libp2p_peerid |> Util.drop_outer_quotes } in let block_producer_configs = - List.mapi block_producers ~f:(fun n node -> + List.map block_producers ~f:(fun node -> if Option.is_none @@ String.Map.find genesis_ledger_accounts node.account_name @@ -253,7 +274,22 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) must be in the genesis ledger. name of Node: %s. name of \ Account which does not exist: %s" node.node_name node.account_name ; - block_producer_config n node.node_name ) + block_producer_config node ) + in + let seed_node_configs = + List.map seed_nodes ~f:(fun node -> + let node_info = + List.find_exn topology ~f:(fun (name, _) -> + String.equal name node.node_name ) + |> snd + in + { name = node.node_name + ; libp2p_pass = "naughty blue worm" + ; libp2p_keypair = node_info.libp2p_keypair + ; libp2p_peerid = + Yojson.Safe.to_string node_info.libp2p_peerid + |> Util.drop_outer_quotes + } ) in let mina_archive_schema = "create_schema.sql" in let long_commit_id = @@ -318,16 +354,13 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ; mina_archive_image = images.archive_node ; runtime_config = Runtime_config.to_yojson runtime_config ; block_producer_configs + ; seed_node_configs ; log_precomputed_blocks ; archive_node_count = List.length archive_nodes ; mina_archive_schema ; mina_archive_schema_aux_files ; snark_coordinator_config ; snark_worker_fee - ; topology = - Test_config.( - topology_of_test_config test_config private_keys libp2p_keypairs - libp2p_peerids - |> Topology.to_yojson) + ; topology = Test_config.Topology.to_yojson topology } } diff --git a/src/lib/integration_test_abstract_engine/unit_tests.ml b/src/lib/integration_test_abstract_engine/unit_tests.ml index fee35e30e0d..22130319e0b 100644 --- a/src/lib/integration_test_abstract_engine/unit_tests.ml +++ b/src/lib/integration_test_abstract_engine/unit_tests.ml @@ -83,6 +83,7 @@ module Test_values = struct let mina_archive_image = "archive" in let runtime_config = `Null in let block_producer_configs = [] in + let seed_node_configs = [] in let log_precomputed_blocks = false in let archive_node_count = 0 in let mina_archive_schema = "schema" in @@ -100,6 +101,7 @@ module Test_values = struct ; mina_archive_image ; runtime_config ; block_producer_configs + ; seed_node_configs ; log_precomputed_blocks ; archive_node_count ; mina_archive_schema @@ -316,6 +318,7 @@ module Request_tests = struct "mina_archive_image": "archive", "runtime_config": "null", "block_producer_configs": [], + "seed_node_configs": [], "log_precomputed_blocks": false, "archive_node_count": 0, "mina_archive_schema": "schema", From 78feb42b8f393b336377cbb7a59d1ba284a73193 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 5 Sep 2023 23:50:49 +0000 Subject: [PATCH 59/90] (intg test) Write libp2p keyfiles to automation dir --- .../mina_automation.ml | 37 ++++++++++++++++--- src/lib/integration_test_lib/test_config.ml | 7 ++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index 96e0f53733c..36005e107cb 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -28,7 +28,7 @@ module Network_manager = struct let create ~logger (network_config : Network_config.t) _test_config = let open Malleable_error.Let_syntax in let testnet_dir = - network_config.config.config_dir ^/ "/testnets" + network_config.config.config_dir ^/ "testnets" ^/ network_config.config.network_id in let%bind () = @@ -85,18 +85,43 @@ module Network_manager = struct Out_channel.with_file ~fail_if_exists:true runtime_config_filename ~f:(fun ch -> network_config.config.runtime_config |> Yojson.Safe.to_channel ch ) ; - [%log info] "Writing topology to %s" topology_filename ; + [%log info] "Writing topology file to %s" topology_filename ; Out_channel.with_file ~fail_if_exists:true topology_filename ~f:(fun ch -> network_config.config.topology |> Yojson.Safe.to_channel ch ) ; - [%log info] "Writing out the genesis keys to testnet dir %s" testnet_dir ; - let kps_base_path = testnet_dir ^ "/genesis_keys" in + let network_kps_base_path = testnet_dir ^/ "genesis_keys" in + let libp2p_kps_base_path = testnet_dir ^/ "libp2p_keys" in let open Deferred.Let_syntax in - let%bind () = Unix.mkdir kps_base_path in + let%bind () = Unix.mkdir network_kps_base_path in + let%bind () = Unix.mkdir libp2p_kps_base_path in + [%log info] "Writing genesis keypairs to %s" network_kps_base_path ; let%bind () = Core.String.Map.iter network_config.genesis_keypairs ~f:(fun kp -> Network_keypair.to_yojson kp |> Yojson.Safe.to_file - (sprintf "%s/%s.json" kps_base_path kp.keypair_name) ) + (sprintf "%s/%s.json" network_kps_base_path kp.keypair_name) ) + |> Deferred.return + in + [%log info] "Writing block producer libp2p keypairs to %s" + libp2p_kps_base_path ; + let%bind () = + List.iter network_config.config.block_producer_configs ~f:(fun config -> + let keypair_file = + sprintf "%s/%s.json" libp2p_kps_base_path config.name + in + Yojson.Safe.to_file keypair_file config.libp2p_keypair ; + Out_channel.write_all (keypair_file ^ ".peerid") + ~data:config.libp2p_peerid ) + |> Deferred.return + in + [%log info] "Writing seed node libp2p keypairs to %s" libp2p_kps_base_path ; + let%bind () = + List.iter network_config.config.seed_node_configs ~f:(fun config -> + let keypair_file = + sprintf "%s/%s.json" libp2p_kps_base_path config.name + in + Yojson.Safe.to_file keypair_file config.libp2p_keypair ; + Out_channel.write_all (keypair_file ^ ".peerid") + ~data:config.libp2p_peerid ) |> Deferred.return in Malleable_error.return t diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 656c8559638..3b05c5d3c98 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -155,6 +155,7 @@ module Topology = struct ; sk : string ; role : Node_role.t ; docker_image : string + ; libp2p_pass : string ; libp2p_keypair : Yojson.Safe.t ; libp2p_peerid : Yojson.Safe.t } @@ -271,6 +272,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : in (pk, sk) in + let libp2p_pass = "naughty blue worm" in let topology_of_block_producer n { Block_producer_node.node_name; docker_image; _ } : string * Topology.node_info = @@ -280,6 +282,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; sk ; role = Block_producer ; docker_image + ; libp2p_pass ; libp2p_keypair = List.nth_exn libp2p_keypairs n ; libp2p_peerid = `String List.(nth_exn libp2p_peerids n) } ) @@ -293,6 +296,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; sk ; role = Snark_coordinator ; docker_image + ; libp2p_pass = "" ; libp2p_keypair = `Null ; libp2p_peerid = `Null } ) @@ -313,6 +317,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; sk ; role = Archive_node ; docker_image + ; libp2p_pass = "" ; libp2p_keypair = `Null ; libp2p_peerid = `Null } ) @@ -326,6 +331,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; sk ; role = Seed_node ; docker_image + ; libp2p_pass ; libp2p_keypair = List.nth_exn libp2p_keypairs n ; libp2p_peerid = `String List.(nth_exn libp2p_peerids n) } ) @@ -340,6 +346,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; sk ; role = Snark_worker ; docker_image + ; libp2p_pass = "" ; libp2p_keypair = `Null ; libp2p_peerid = `Null } ) From cc3bbfbee37c410b5dad099382d3f2a76d5f285d Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 6 Sep 2023 17:49:19 +0000 Subject: [PATCH 60/90] (intg test) Separate base, archive, and node topology info --- src/lib/integration_test_lib/test_config.ml | 145 ++++++++++---------- 1 file changed, 70 insertions(+), 75 deletions(-) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 3b05c5d3c98..983e8c8f5d1 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -150,21 +150,49 @@ module Node_role = struct end module Topology = struct + type base_info = + { pk : string; sk : string; role : Node_role.t; docker_image : string } + [@@deriving to_yojson] + + type archive_info = + { pk : string + ; sk : string + ; role : Node_role.t + ; docker_image : string + ; schema_file : string + ; zkapp_file : string + } + [@@deriving to_yojson] + type node_info = { pk : string ; sk : string ; role : Node_role.t ; docker_image : string ; libp2p_pass : string + ; libp2p_keyfile : string ; libp2p_keypair : Yojson.Safe.t ; libp2p_peerid : Yojson.Safe.t } [@@deriving to_yojson] - type t = (string * node_info) list + type top_info = + | Base of string * base_info + | Archive of string * archive_info + | Node of string * node_info + + type t = top_info list let to_yojson nodes : Yojson.Safe.t = - let alist = List.map nodes ~f:(fun (a, b) -> (a, node_info_to_yojson b)) in + let alist = + List.map nodes ~f:(function + | Base (a, b) -> + (a, base_info_to_yojson b) + | Archive (a, b) -> + (a, archive_info_to_yojson b) + | Node (a, b) -> + (a, node_info_to_yojson b) ) + in `Assoc alist end @@ -274,32 +302,25 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : in let libp2p_pass = "naughty blue worm" in let topology_of_block_producer n - { Block_producer_node.node_name; docker_image; _ } : - string * Topology.node_info = + { Block_producer_node.node_name; docker_image; _ } : Topology.top_info = let pk, sk = pk_sk n in - ( node_name - , { pk - ; sk - ; role = Block_producer - ; docker_image - ; libp2p_pass - ; libp2p_keypair = List.nth_exn libp2p_keypairs n - ; libp2p_peerid = `String List.(nth_exn libp2p_peerids n) - } ) + Node + ( node_name + , { pk + ; sk + ; role = Block_producer + ; docker_image + ; libp2p_pass + ; libp2p_keyfile = "" (* value set in network_config.ml*) + ; libp2p_keypair = List.nth_exn libp2p_keypairs n + ; libp2p_peerid = `String List.(nth_exn libp2p_peerids n) + } ) in let topology_of_snark_coordinator - { Snark_coordinator_node.node_name; docker_image; _ } : - string * Topology.node_info = + { Snark_coordinator_node.node_name; docker_image; _ } : Topology.top_info + = let pk, sk = pk_sk num_bp in - ( node_name - , { pk - ; sk - ; role = Snark_coordinator - ; docker_image - ; libp2p_pass = "" - ; libp2p_keypair = `Null - ; libp2p_peerid = `Null - } ) + Base (node_name, { pk; sk; role = Snark_coordinator; docker_image }) in let snark_coordinator = match Option.map t.snark_coordinator ~f:topology_of_snark_coordinator with @@ -309,47 +330,40 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : [ sc ] in let topology_of_archive n { Archive_node.node_name; docker_image; _ } : - string * Topology.node_info = + Topology.top_info = let n = n + num_bp_sc in let pk, sk = pk_sk n in - ( node_name - , { pk - ; sk - ; role = Archive_node - ; docker_image - ; libp2p_pass = "" - ; libp2p_keypair = `Null - ; libp2p_peerid = `Null - } ) + Archive + ( node_name + , { pk + ; sk + ; role = Archive_node + ; docker_image + ; schema_file = "" + ; zkapp_file = "" + } ) in let topology_of_seed n { Seed_node.node_name; docker_image; _ } : - string * Topology.node_info = + Topology.top_info = let n = n + num_bp_sc_an in let pk, sk = pk_sk n in - ( node_name - , { pk - ; sk - ; role = Seed_node - ; docker_image - ; libp2p_pass - ; libp2p_keypair = List.nth_exn libp2p_keypairs n - ; libp2p_peerid = `String List.(nth_exn libp2p_peerids n) - } ) + Node + ( node_name + , { pk + ; sk + ; role = Seed_node + ; docker_image + ; libp2p_pass + ; libp2p_keyfile = "" (* value set in network_config.ml*) + ; libp2p_keypair = List.nth_exn libp2p_keypairs n + ; libp2p_peerid = `String List.(nth_exn libp2p_peerids n) + } ) in let topology_of_snark_worker n - { Snark_worker_node.node_name; docker_image; _ } : - string * Topology.node_info = + { Snark_worker_node.node_name; docker_image; _ } : Topology.top_info = let n = n + num_bp_sc_an_sn in let pk, sk = pk_sk n in - ( node_name - , { pk - ; sk - ; role = Snark_worker - ; docker_image - ; libp2p_pass = "" - ; libp2p_keypair = `Null - ; libp2p_peerid = `Null - } ) + Base (node_name, { pk; sk; role = Snark_worker; docker_image }) in snark_coordinator @ List.mapi t.archive_nodes ~f:topology_of_archive @@ -560,24 +574,5 @@ module Unit_tests = struct |> Topology.to_yojson in print_endline "=== Topology ===" ; - topology |> pretty_to_string |> print_endline ; - (* only block producers and seed nodes get libp2p keypairs *) - let get_libp2p_keypair node_name = - Util.(member node_name topology |> member "libp2p_keypair") - in - assert ( - List.for_all test_config.block_producers ~f:(fun bp -> - not @@ equal `Null @@ get_libp2p_keypair bp.node_name ) ) ; - assert ( - List.for_all test_config.seed_nodes ~f:(fun sn -> - not @@ equal `Null @@ get_libp2p_keypair sn.node_name ) ) ; - assert ( - List.for_all test_config.archive_nodes ~f:(fun an -> - equal `Null @@ get_libp2p_keypair an.node_name ) ) ; - assert ( - List.for_all test_config.snark_workers ~f:(fun sw -> - equal `Null @@ get_libp2p_keypair sw.node_name ) ) ; - assert ( - Option.for_all test_config.snark_coordinator ~f:(fun sc -> - equal `Null @@ get_libp2p_keypair sc.node_name ) ) + topology |> pretty_to_string |> print_endline end From 2bdbb426d29698d5fa14c58d2e8b00bd9cfa2b7c Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 6 Sep 2023 17:50:37 +0000 Subject: [PATCH 61/90] (intg test) Add libp2p_keyfile path to topology --- .../network_config.ml | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/network_config.ml b/src/lib/integration_test_abstract_engine/network_config.ml index ad53f52c2cc..3bb7188eed9 100644 --- a/src/lib/integration_test_abstract_engine/network_config.ml +++ b/src/lib/integration_test_abstract_engine/network_config.ml @@ -249,10 +249,12 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) let _, (keypair, _) = Core.String.Map.find_exn genesis_ledger_accounts node.account_name in - let node_info = - List.find_exn topology ~f:(fun (name, _) -> - String.equal name node.node_name ) - |> snd + let[@warning "-8"] (Test_config.Topology.Node (_, node_info)) = + List.find_exn topology ~f:(function + | Node (name, _) -> + String.equal name node.node_name + | _ -> + false ) in { name = node.node_name ; keypair = { keypair with keypair_name = node.node_name } @@ -278,10 +280,12 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) in let seed_node_configs = List.map seed_nodes ~f:(fun node -> - let node_info = - List.find_exn topology ~f:(fun (name, _) -> - String.equal name node.node_name ) - |> snd + let[@warning "-8"] (Test_config.Topology.Node (_, node_info)) = + List.find_exn topology ~f:(function + | Node (name, _) -> + String.equal name node.node_name + | _ -> + false ) in { name = node.node_name ; libp2p_pass = "naughty blue worm" @@ -301,10 +305,24 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) "https://raw.githubusercontent.com/MinaProtocol/mina/" ^ long_commit_id ^ "/src/app/archive/" in - let mina_archive_schema_aux_files = - [ mina_archive_base_url ^ "create_schema.sql" - ; mina_archive_base_url ^ "zkapp_tables.sql" - ] + let schema_file = mina_archive_base_url ^ "create_schema.sql" in + let zkapp_file = mina_archive_base_url ^ "zkapp_tables.sql" in + let mina_archive_schema_aux_files = [ schema_file; zkapp_file ] in + let libp2p_keyfile name = + let open Core in + cli_inputs.mina_automation_location ^/ "testnets" ^/ network_id + ^/ "libp2p_keys" ^/ name ^ ".json" + in + (* instantiate remaining topology information *) + let topology = + List.map topology ~f:(function + | Archive (name, archive_info) -> + Test_config.Topology.Archive + (name, { archive_info with schema_file; zkapp_file }) + | Node (name, node_info) -> + Node (name, { node_info with libp2p_keyfile = libp2p_keyfile name }) + | x -> + x ) in let genesis_keypairs = String.Map.of_alist_exn From c7e03c9db0b144b61deae99099e47e7e31301cbf Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 6 Sep 2023 18:44:40 +0000 Subject: [PATCH 62/90] (intg test) Add snark coordinator worker_nodes and snark_worker_fee to topology --- src/lib/integration_test_lib/test_config.ml | 40 +++++++++++++++------ 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 983e8c8f5d1..3b981343849 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -150,7 +150,16 @@ module Node_role = struct end module Topology = struct - type base_info = + type snark_coordinator_info = + { pk : string + ; sk : string + ; role : Node_role.t + ; docker_image : string + ; worker_nodes : int + } + [@@deriving to_yojson] + + type snark_worker_info = { pk : string; sk : string; role : Node_role.t; docker_image : string } [@@deriving to_yojson] @@ -177,21 +186,27 @@ module Topology = struct [@@deriving to_yojson] type top_info = - | Base of string * base_info | Archive of string * archive_info | Node of string * node_info + | Snark_coordinator of string * snark_coordinator_info + | Snark_worker of string * snark_worker_info + | Snark_worker_fee of string type t = top_info list let to_yojson nodes : Yojson.Safe.t = let alist = List.map nodes ~f:(function - | Base (a, b) -> - (a, base_info_to_yojson b) | Archive (a, b) -> (a, archive_info_to_yojson b) | Node (a, b) -> - (a, node_info_to_yojson b) ) + (a, node_info_to_yojson b) + | Snark_coordinator (a, b) -> + (a, snark_coordinator_info_to_yojson b) + | Snark_worker (a, b) -> + (a, snark_worker_info_to_yojson b) + | Snark_worker_fee fee -> + ("snark_worker_fee", `String fee) ) in `Assoc alist end @@ -317,10 +332,12 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : } ) in let topology_of_snark_coordinator - { Snark_coordinator_node.node_name; docker_image; _ } : Topology.top_info - = + { Snark_coordinator_node.node_name; docker_image; worker_nodes; _ } : + Topology.top_info = let pk, sk = pk_sk num_bp in - Base (node_name, { pk; sk; role = Snark_coordinator; docker_image }) + Snark_coordinator + ( node_name + , { pk; sk; role = Snark_coordinator; docker_image; worker_nodes } ) in let snark_coordinator = match Option.map t.snark_coordinator ~f:topology_of_snark_coordinator with @@ -339,8 +356,8 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; sk ; role = Archive_node ; docker_image - ; schema_file = "" - ; zkapp_file = "" + ; schema_file = "" (* value set in network_config.ml*) + ; zkapp_file = "" (* value set in network_config.ml*) } ) in let topology_of_seed n { Seed_node.node_name; docker_image; _ } : @@ -363,13 +380,14 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : { Snark_worker_node.node_name; docker_image; _ } : Topology.top_info = let n = n + num_bp_sc_an_sn in let pk, sk = pk_sk n in - Base (node_name, { pk; sk; role = Snark_worker; docker_image }) + Snark_worker (node_name, { pk; sk; role = Snark_worker; docker_image }) in snark_coordinator @ List.mapi t.archive_nodes ~f:topology_of_archive @ List.mapi t.block_producers ~f:topology_of_block_producer @ List.mapi t.seed_nodes ~f:topology_of_seed @ List.mapi t.snark_workers ~f:topology_of_snark_worker + @ [ Snark_worker_fee t.snark_worker_fee ] let test_account ?(pk = "") ?(timing = Mina_base.Account.Timing.Untimed) account_name balance : Test_Account.t = From adaa7818e9c59f919385f441fdbdfe47d6da4611 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 6 Sep 2023 19:11:00 +0000 Subject: [PATCH 63/90] (intg test) Rename --runtime-config to --genesis-ledger --- integration_tests/mock_config.json | 4 ++-- .../mina_automation.ml | 12 ++++++------ src/lib/integration_test_lib/test_config.ml | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/integration_tests/mock_config.json b/integration_tests/mock_config.json index 62052374fff..9dfa9b52c14 100644 --- a/integration_tests/mock_config.json +++ b/integration_tests/mock_config.json @@ -5,10 +5,10 @@ "name": "create_network", "args": { "network_id": "string", - "runtime_config": "string", + "genesis_ledger": "string", "topology": "string" }, - "command": "MOCK_NETWORK network create --network-id {{network_id}} --runtime-config {{runtime_config}} --topology {{topology}}" + "command": "MOCK_NETWORK network create --network-id {{network_id}} --genesis-ledger {{genesis_ledger}} --topology {{topology}}" }, { "name": "start_network", diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index 36005e107cb..93385c3ba1c 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -14,7 +14,7 @@ module Network_manager = struct ; constants : Test_config.constants ; mutable deployed : bool ; genesis_keypairs : Network_keypair.t Core.String.Map.t - ; runtime_config_path : string + ; genesis_ledger_path : string ; topology_path : string } @@ -62,7 +62,7 @@ module Network_manager = struct [%log info] "Making new testnet dir %s" testnet_dir ; mkdir_p testnet_dir ; let network_config_filename = testnet_dir ^/ "network_config.json" in - let runtime_config_filename = testnet_dir ^/ "runtime_config.json" in + let genesis_ledger_filename = testnet_dir ^/ "genesis_ledger.json" in let topology_filename = testnet_dir ^/ "topology.json" in let t = { logger @@ -73,7 +73,7 @@ module Network_manager = struct ; constants = network_config.constants ; deployed = false ; genesis_keypairs = network_config.genesis_keypairs - ; runtime_config_path = runtime_config_filename + ; genesis_ledger_path = genesis_ledger_filename ; topology_path = topology_filename } in @@ -81,8 +81,8 @@ module Network_manager = struct Out_channel.with_file ~fail_if_exists:true network_config_filename ~f:(fun ch -> Network_config.to_yojson network_config |> Yojson.Safe.to_channel ch ) ; - [%log info] "Writing runtime configuration to %s" runtime_config_filename ; - Out_channel.with_file ~fail_if_exists:true runtime_config_filename + [%log info] "Writing runtime configuration to %s" genesis_ledger_filename ; + Out_channel.with_file ~fail_if_exists:true genesis_ledger_filename ~f:(fun ch -> network_config.config.runtime_config |> Yojson.Safe.to_channel ch ) ; [%log info] "Writing topology file to %s" topology_filename ; @@ -137,7 +137,7 @@ module Network_manager = struct ~config:!Abstract_network.config_path ~args: [ ("network_id", `String t.network_id) - ; ("runtime_config", `String t.runtime_config_path) + ; ("genesis_ledger", `String t.genesis_ledger_path) ; ("topology", `String t.topology_path) ] "create_network" diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 3b981343849..cae8038715c 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -326,7 +326,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; role = Block_producer ; docker_image ; libp2p_pass - ; libp2p_keyfile = "" (* value set in network_config.ml*) + ; libp2p_keyfile = "instantiated in network_config.ml" ; libp2p_keypair = List.nth_exn libp2p_keypairs n ; libp2p_peerid = `String List.(nth_exn libp2p_peerids n) } ) @@ -356,8 +356,8 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; sk ; role = Archive_node ; docker_image - ; schema_file = "" (* value set in network_config.ml*) - ; zkapp_file = "" (* value set in network_config.ml*) + ; schema_file = "instantiated in network_config.ml" + ; zkapp_file = "instantiated in network_config.ml" } ) in let topology_of_seed n { Seed_node.node_name; docker_image; _ } : @@ -371,7 +371,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; role = Seed_node ; docker_image ; libp2p_pass - ; libp2p_keyfile = "" (* value set in network_config.ml*) + ; libp2p_keyfile = "instantiated in network_config.ml" ; libp2p_keypair = List.nth_exn libp2p_keypairs n ; libp2p_peerid = `String List.(nth_exn libp2p_peerids n) } ) From e43199d8726f78ddd041db2a38a6112a2c7be575 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 6 Sep 2023 22:15:38 +0000 Subject: [PATCH 64/90] (intg test) Sync keys in genesis ledger and topology --- src/lib/integration_test_lib/test_config.ml | 40 +++++++++++++-------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index cae8038715c..2494f3a5d1b 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -305,7 +305,8 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : in let num_bp_sc_an = num_bp_sc + List.length t.archive_nodes in let num_bp_sc_an_sn = num_bp_sc_an + List.length t.seed_nodes in - let pk_sk n = + let keypairs = ref Core.String.Map.empty in + let pk_sk name n = let open Signature_lib in let sk = List.nth_exn private_keys n in let pk = @@ -313,12 +314,18 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : Private_key.of_base58_check_exn sk |> of_private_key_exn |> compress |> Compressed.to_base58_check in - (pk, sk) + match Core.String.Map.find !keypairs name with + | Some keys -> + keys + | None -> + keypairs := Core.String.Map.add_exn !keypairs ~key:name ~data:(pk, sk) ; + (pk, sk) in let libp2p_pass = "naughty blue worm" in let topology_of_block_producer n - { Block_producer_node.node_name; docker_image; _ } : Topology.top_info = - let pk, sk = pk_sk n in + { Block_producer_node.account_name; node_name; docker_image } : + Topology.top_info = + let pk, sk = pk_sk account_name n in Node ( node_name , { pk @@ -332,9 +339,12 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : } ) in let topology_of_snark_coordinator - { Snark_coordinator_node.node_name; docker_image; worker_nodes; _ } : - Topology.top_info = - let pk, sk = pk_sk num_bp in + { Snark_coordinator_node.account_name + ; node_name + ; docker_image + ; worker_nodes + } : Topology.top_info = + let pk, sk = pk_sk account_name num_bp in Snark_coordinator ( node_name , { pk; sk; role = Snark_coordinator; docker_image; worker_nodes } ) @@ -346,10 +356,11 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : | Some sc -> [ sc ] in - let topology_of_archive n { Archive_node.node_name; docker_image; _ } : - Topology.top_info = + let topology_of_archive n + { Archive_node.account_name; node_name; docker_image } : Topology.top_info + = let n = n + num_bp_sc in - let pk, sk = pk_sk n in + let pk, sk = pk_sk account_name n in Archive ( node_name , { pk @@ -360,10 +371,10 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; zkapp_file = "instantiated in network_config.ml" } ) in - let topology_of_seed n { Seed_node.node_name; docker_image; _ } : + let topology_of_seed n { Seed_node.account_name; node_name; docker_image } : Topology.top_info = let n = n + num_bp_sc_an in - let pk, sk = pk_sk n in + let pk, sk = pk_sk account_name n in Node ( node_name , { pk @@ -377,9 +388,10 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : } ) in let topology_of_snark_worker n - { Snark_worker_node.node_name; docker_image; _ } : Topology.top_info = + { Snark_worker_node.account_name; node_name; docker_image } : + Topology.top_info = let n = n + num_bp_sc_an_sn in - let pk, sk = pk_sk n in + let pk, sk = pk_sk account_name n in Snark_worker (node_name, { pk; sk; role = Snark_worker; docker_image }) in snark_coordinator From fc58161469ce33c8f10f8543cd7e29c5b11ba8d8 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Wed, 6 Sep 2023 22:46:28 +0000 Subject: [PATCH 65/90] (intg test) Small corrections to keypairs and return spec --- integration_tests/return_spec.json | 12 +++++------- .../abstract_network.ml | 4 +--- .../integration_test_abstract_engine/config_util.ml | 7 +++---- .../integration_test_abstract_engine/unit_tests.ml | 2 +- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/integration_tests/return_spec.json b/integration_tests/return_spec.json index fc111f1efb9..a6a5d8d4a6d 100644 --- a/integration_tests/return_spec.json +++ b/integration_tests/return_spec.json @@ -3,15 +3,13 @@ "return_spec": { "create_network": { "network_id": "string", - "nodes": [ - { + "nodes": { + "$node_id": { "graphql_uri" : "string", - "network_id" : "string", - "network_keypair" : "string", - "node_id" : "string", - "node_type" : "string" + "node_type" : "string", + "private_key" : "string" } - ] + } }, "start_network": { "network_id": "string" diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index ba30500293e..59e0f0effa2 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -108,9 +108,7 @@ module Node = struct module Scalars = Graphql_lib.Scalars module Graphql = struct - let ingress_uri node = - let path = sprintf "/%s/graphql" node.graphql_uri in - Uri.make ~scheme:"http" ~path ~port:80 () + let ingress_uri node = Uri.of_string node.graphql_uri module Client = Graphql_lib.Client.Make (struct let preprocess_variables_string = Fn.id diff --git a/src/lib/integration_test_abstract_engine/config_util.ml b/src/lib/integration_test_abstract_engine/config_util.ml index 75ccde61d20..51968ee59c1 100644 --- a/src/lib/integration_test_abstract_engine/config_util.ml +++ b/src/lib/integration_test_abstract_engine/config_util.ml @@ -109,8 +109,7 @@ module Network_deployed = struct (fun (m : Network_keypair.t) n -> Public_key.equal m.keypair.public_key n.keypair.public_key && String.equal m.public_key n.public_key - && String.equal m.privkey_password n.privkey_password - && String.equal m.keypair_name n.keypair_name ) + && String.equal m.privkey_password n.privkey_password ) m.network_keypair n.network_keypair in let graphql = Option.equal String.equal m.graphql_uri n.graphql_uri in @@ -137,12 +136,12 @@ module Network_deployed = struct | `Null -> None | `String private_key -> - let keypair_name = node_id ^ "_key" in let keypair = Keypair.of_private_key_exn @@ Private_key.of_base58_check_exn private_key in - Some (Network_keypair.create_network_keypair ~keypair_name ~keypair) + Some + (Network_keypair.create_network_keypair ~keypair_name:node_id ~keypair) | _ -> raise @@ Invalid_keypair private_key diff --git a/src/lib/integration_test_abstract_engine/unit_tests.ml b/src/lib/integration_test_abstract_engine/unit_tests.ml index 22130319e0b..dba22e18ea4 100644 --- a/src/lib/integration_test_abstract_engine/unit_tests.ml +++ b/src/lib/integration_test_abstract_engine/unit_tests.ml @@ -390,7 +390,7 @@ module Parse_output_tests = struct let public_key = Public_key.of_private_key_exn private_key in Keypair.{ public_key; private_key } in - Network_keypair.create_network_keypair ~keypair_name:"node2_key" ~keypair + Network_keypair.create_network_keypair ~keypair_name:"node2" ~keypair in let archive = { node_id = "node0" From 8d46dfd4f004bb06a6d41be7c3d16048585df55c Mon Sep 17 00:00:00 2001 From: Quantifier Date: Thu, 7 Sep 2023 15:07:12 +0000 Subject: [PATCH 66/90] (intg test) Remove snark worker objects from topology --- src/app/test_executive/mock.ml | 22 +++++++-------- .../network_config.ml | 5 +--- src/lib/integration_test_lib/test_config.ml | 28 +------------------ 3 files changed, 12 insertions(+), 43 deletions(-) diff --git a/src/app/test_executive/mock.ml b/src/app/test_executive/mock.ml index e5b5849e6c7..920b2024405 100644 --- a/src/app/test_executive/mock.ml +++ b/src/app/test_executive/mock.ml @@ -22,12 +22,10 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let i_str = Int.to_string i in test_account ("sender-account-" ^ i_str) "10000" ) ; block_producers = - [ bp "receiver" "another-docker-image" + [ bp "receiver" "receiver-docker-image" ; bp "empty_node-1" ~account_name:"empty-bp-key" !Network.mina_image ; bp "empty_node-2" ~account_name:"empty-bp-key" !Network.mina_image - ; bp "empty_node-3" ~account_name:"empty-bp-key" !Network.mina_image - ; bp "empty_node-4" ~account_name:"empty-bp-key" !Network.mina_image - ; bp "observer" ~account_name:"empty-bp-key" !Network.mina_image + ; bp "observer" ~account_name:"empty-bp-key" "observer-docker-image" ] ; snark_coordinator = Some @@ -49,17 +47,17 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ; account_name = "seed-0-key" ; docker_image = "seed-docker-image" } - ] - ; snark_workers = - [ { node_name = "snark-0" - ; account_name = "snark-0-key" - ; docker_image = "snark-docker-image" + ; { node_name = "seed-1" + ; account_name = "seed-1-key" + ; docker_image = !Network.mina_image } ] ; archive_nodes = - [ { node_name = "archive-0" - ; account_name = "archive-0-key" - ; docker_image = "archive-docker-image" + [ { node_name = "archive-node" + ; account_name = "archive-node-key" + ; docker_image = + Option.value !Network.archive_image + ~default:"default-archive-image" } ] } diff --git a/src/lib/integration_test_abstract_engine/network_config.ml b/src/lib/integration_test_abstract_engine/network_config.ml index 3bb7188eed9..3b5bf71a767 100644 --- a/src/lib/integration_test_abstract_engine/network_config.ml +++ b/src/lib/integration_test_abstract_engine/network_config.ml @@ -88,7 +88,6 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ; slots_per_sub_window ; txpool_max_size ; seed_nodes - ; snark_workers } = test_config in @@ -121,9 +120,7 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) let num_keypairs = List.length genesis_ledger in let network_keypairs, private_keys, libp2p_keypairs, libp2p_peerids = let max_num_nodes = - List.length archive_nodes - + List.length block_producers - + List.length seed_nodes + List.length snark_workers + 1 + List.length archive_nodes + List.length block_producers + 1 in Util.pull_keypairs !keypairs_path (List.length genesis_ledger + max_num_nodes) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 2494f3a5d1b..167c58cd01a 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -97,11 +97,6 @@ module Snark_coordinator_node = struct [@@deriving to_yojson] end -module Snark_worker_node = struct - type t = { node_name : string; account_name : string; docker_image : string } - [@@deriving to_yojson] -end - type constants = { constraints : Genesis_constants.Constraint_constants.t ; genesis : Genesis_constants.t @@ -116,7 +111,6 @@ type t = ; block_producers : Block_producer_node.t list ; seed_nodes : Seed_node.t list ; snark_coordinator : Snark_coordinator_node.t option - ; snark_workers : Snark_worker_node.t list ; snark_worker_fee : string ; log_precomputed_blocks : bool ; proof_config : Runtime_config.Proof_keys.t @@ -129,12 +123,7 @@ type t = [@@deriving to_yojson] module Node_role = struct - type t = - | Archive_node - | Block_producer - | Seed_node - | Snark_coordinator - | Snark_worker + type t = Archive_node | Block_producer | Seed_node | Snark_coordinator let to_yojson = function | Archive_node -> @@ -145,8 +134,6 @@ module Node_role = struct `String "Seed_node" | Snark_coordinator -> `String "Snark_coordinator" - | Snark_worker -> - `String "Snark_worker" end module Topology = struct @@ -189,7 +176,6 @@ module Topology = struct | Archive of string * archive_info | Node of string * node_info | Snark_coordinator of string * snark_coordinator_info - | Snark_worker of string * snark_worker_info | Snark_worker_fee of string type t = top_info list @@ -203,8 +189,6 @@ module Topology = struct (a, node_info_to_yojson b) | Snark_coordinator (a, b) -> (a, snark_coordinator_info_to_yojson b) - | Snark_worker (a, b) -> - (a, snark_worker_info_to_yojson b) | Snark_worker_fee fee -> ("snark_worker_fee", `String fee) ) in @@ -234,7 +218,6 @@ let default = ; block_producers = [] ; seed_nodes = [] ; snark_coordinator = None - ; snark_workers = [] ; snark_worker_fee = "0.025" ; log_precomputed_blocks = false ; proof_config = proof_config_default @@ -304,7 +287,6 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : num_bp + if Option.is_some t.snark_coordinator then 1 else 0 in let num_bp_sc_an = num_bp_sc + List.length t.archive_nodes in - let num_bp_sc_an_sn = num_bp_sc_an + List.length t.seed_nodes in let keypairs = ref Core.String.Map.empty in let pk_sk name n = let open Signature_lib in @@ -387,18 +369,10 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; libp2p_peerid = `String List.(nth_exn libp2p_peerids n) } ) in - let topology_of_snark_worker n - { Snark_worker_node.account_name; node_name; docker_image } : - Topology.top_info = - let n = n + num_bp_sc_an_sn in - let pk, sk = pk_sk account_name n in - Snark_worker (node_name, { pk; sk; role = Snark_worker; docker_image }) - in snark_coordinator @ List.mapi t.archive_nodes ~f:topology_of_archive @ List.mapi t.block_producers ~f:topology_of_block_producer @ List.mapi t.seed_nodes ~f:topology_of_seed - @ List.mapi t.snark_workers ~f:topology_of_snark_worker @ [ Snark_worker_fee t.snark_worker_fee ] let test_account ?(pk = "") ?(timing = Mina_base.Account.Timing.Untimed) From 96695f0cd71d7e6e135972463799882a5735c5cf Mon Sep 17 00:00:00 2001 From: Quantifier Date: Thu, 7 Sep 2023 16:01:58 +0000 Subject: [PATCH 67/90] (intg test) Rework archive and coordinator topology --- .../network_config.ml | 6 +++--- src/lib/integration_test_lib/test_config.ml | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/network_config.ml b/src/lib/integration_test_abstract_engine/network_config.ml index 3b5bf71a767..0996253fd9a 100644 --- a/src/lib/integration_test_abstract_engine/network_config.ml +++ b/src/lib/integration_test_abstract_engine/network_config.ml @@ -303,8 +303,8 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ^ "/src/app/archive/" in let schema_file = mina_archive_base_url ^ "create_schema.sql" in - let zkapp_file = mina_archive_base_url ^ "zkapp_tables.sql" in - let mina_archive_schema_aux_files = [ schema_file; zkapp_file ] in + let zkapp_table = mina_archive_base_url ^ "zkapp_tables.sql" in + let mina_archive_schema_aux_files = [ schema_file; zkapp_table ] in let libp2p_keyfile name = let open Core in cli_inputs.mina_automation_location ^/ "testnets" ^/ network_id @@ -315,7 +315,7 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) List.map topology ~f:(function | Archive (name, archive_info) -> Test_config.Topology.Archive - (name, { archive_info with schema_file; zkapp_file }) + (name, { archive_info with schema_file; zkapp_table }) | Node (name, node_info) -> Node (name, { node_info with libp2p_keyfile = libp2p_keyfile name }) | x -> diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 167c58cd01a..825358e63d1 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -143,6 +143,7 @@ module Topology = struct ; role : Node_role.t ; docker_image : string ; worker_nodes : int + ; snark_worker_fee : string } [@@deriving to_yojson] @@ -156,7 +157,7 @@ module Topology = struct ; role : Node_role.t ; docker_image : string ; schema_file : string - ; zkapp_file : string + ; zkapp_table : string } [@@deriving to_yojson] @@ -176,7 +177,6 @@ module Topology = struct | Archive of string * archive_info | Node of string * node_info | Snark_coordinator of string * snark_coordinator_info - | Snark_worker_fee of string type t = top_info list @@ -188,9 +188,7 @@ module Topology = struct | Node (a, b) -> (a, node_info_to_yojson b) | Snark_coordinator (a, b) -> - (a, snark_coordinator_info_to_yojson b) - | Snark_worker_fee fee -> - ("snark_worker_fee", `String fee) ) + (a, snark_coordinator_info_to_yojson b) ) in `Assoc alist end @@ -329,7 +327,13 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : let pk, sk = pk_sk account_name num_bp in Snark_coordinator ( node_name - , { pk; sk; role = Snark_coordinator; docker_image; worker_nodes } ) + , { pk + ; sk + ; role = Snark_coordinator + ; docker_image + ; worker_nodes + ; snark_worker_fee = t.snark_worker_fee + } ) in let snark_coordinator = match Option.map t.snark_coordinator ~f:topology_of_snark_coordinator with @@ -350,7 +354,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; role = Archive_node ; docker_image ; schema_file = "instantiated in network_config.ml" - ; zkapp_file = "instantiated in network_config.ml" + ; zkapp_table = "instantiated in network_config.ml" } ) in let topology_of_seed n { Seed_node.account_name; node_name; docker_image } : @@ -373,7 +377,6 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : @ List.mapi t.archive_nodes ~f:topology_of_archive @ List.mapi t.block_producers ~f:topology_of_block_producer @ List.mapi t.seed_nodes ~f:topology_of_seed - @ [ Snark_worker_fee t.snark_worker_fee ] let test_account ?(pk = "") ?(timing = Mina_base.Account.Timing.Untimed) account_name balance : Test_Account.t = From d9d32d76314a6e26b5a18fb4d4fe7a4230a7dc6b Mon Sep 17 00:00:00 2001 From: Quantifier Date: Thu, 7 Sep 2023 18:11:42 +0000 Subject: [PATCH 68/90] (intg test) Check same keys, different libp2p keys for node with same account --- src/lib/integration_test_lib/test_config.ml | 25 ++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 825358e63d1..4af39f95fbc 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -391,16 +391,20 @@ module Unit_tests = struct genesis_ledger = [ test_account "receiver" "9999999" ; test_account "empty-bp" "0" - ; test_account "snark-node-key" "0" + ; test_account "snark-node" "0" ] @ List.init 2 ~f:(fun i -> test_account (sprintf "sender-account-%d" i) "10000" ) ; block_producers = [ { node_name = "receiver" ; account_name = "receiver-key" + ; docker_image = "bp-image" + } + ; { node_name = "empty-node-0" + ; account_name = "empty-bp-key" ; docker_image = "bp-image-0" } - ; { node_name = "empty-node" + ; { node_name = "empty-node-1" ; account_name = "empty-bp-key" ; docker_image = "bp-image-1" } @@ -581,5 +585,20 @@ module Unit_tests = struct |> Topology.to_yojson in print_endline "=== Topology ===" ; - topology |> pretty_to_string |> print_endline + topology |> pretty_to_string |> print_endline ; + let empty_node_0 = Util.member "empty-node-0" topology in + let empty_node_1 = Util.member "empty-node-1" topology in + let pk_0 = Util.member "pk" empty_node_0 in + let sk_0 = Util.member "sk" empty_node_0 in + let kp_0 = Util.member "libp2p_keypair" empty_node_0 in + let pk_1 = Util.member "pk" empty_node_1 in + let sk_1 = Util.member "sk" empty_node_1 in + let kp_1 = Util.member "libp2p_keypair" empty_node_1 in + let ( = ) = equal in + assert ( + (* same public + private keys *) + (not (pk_0 = `Null)) + && pk_0 = pk_1 && sk_0 = sk_1 + (* different libp2p keys *) + && not (kp_0 = kp_1) ) end From 3da569ddebe2e8a58a183205c24b8f75c9014f56 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Thu, 7 Sep 2023 22:16:17 +0000 Subject: [PATCH 69/90] (intg test) Write compatible block producer keys --- .../mina_automation.ml | 12 ++++++++++++ src/lib/integration_test_lib/util.ml | 17 ++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index 93385c3ba1c..82c9286ec89 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -90,9 +90,11 @@ module Network_manager = struct network_config.config.topology |> Yojson.Safe.to_channel ch ) ; let network_kps_base_path = testnet_dir ^/ "genesis_keys" in let libp2p_kps_base_path = testnet_dir ^/ "libp2p_keys" in + let bp_keys_base_path = testnet_dir ^/ "block_producer_keys" in let open Deferred.Let_syntax in let%bind () = Unix.mkdir network_kps_base_path in let%bind () = Unix.mkdir libp2p_kps_base_path in + let%bind () = Unix.mkdir bp_keys_base_path in [%log info] "Writing genesis keypairs to %s" network_kps_base_path ; let%bind () = Core.String.Map.iter network_config.genesis_keypairs ~f:(fun kp -> @@ -101,6 +103,16 @@ module Network_manager = struct (sprintf "%s/%s.json" network_kps_base_path kp.keypair_name) ) |> Deferred.return in + [%log info] "Writing block producer private keys to %s" bp_keys_base_path ; + let%bind () = + List.iter network_config.config.block_producer_configs ~f:(fun config -> + let keypair_file = + sprintf "%s/%s.json" bp_keys_base_path config.name + in + Yojson.Safe.from_string config.keypair.private_key + |> Yojson.Safe.to_file keypair_file ) + |> Deferred.return + in [%log info] "Writing block producer libp2p keypairs to %s" libp2p_kps_base_path ; let%bind () = diff --git a/src/lib/integration_test_lib/util.ml b/src/lib/integration_test_lib/util.ml index 5f9f2225657..c24faa9489c 100644 --- a/src/lib/integration_test_lib/util.ml +++ b/src/lib/integration_test_lib/util.ml @@ -34,20 +34,19 @@ let pull_keypairs path num_keypairs = |> generate ~size:num_keypairs ~random in (* network keypairs + private keys *) - let base_filename n = - path ^/ sprintf "network-keypairs/sender-account-%d.json" n - in + let keypair_name = sprintf "network-keypairs/network-keypair-%d" in + let base_filename n = path ^/ keypair_name n ^ ".json" in let read_sk n = In_channel.read_all (base_filename n ^ ".sk") in let read_keypair n = - let open Yojson.Safe in let open Signature_lib in - let json = from_file (base_filename n) in let sk = read_sk n |> Private_key.of_base58_check_exn in { Network_keypair.keypair = Keypair.of_private_key_exn sk - ; keypair_name = Util.member "keypair_name" json |> to_string - ; privkey_password = Util.member "privkey_password" json |> to_string - ; public_key = Util.member "public_key" json |> to_string - ; private_key = Util.member "private_key" json |> to_string + ; keypair_name = keypair_name n + ; privkey_password = "naughty blue worm" + ; public_key = + Public_key.( + of_private_key_exn sk |> compress |> Compressed.to_base58_check) + ; private_key = In_channel.read_all (base_filename n) } in (* libp2p keypairs *) From 974e1ec57867a6cbf4c34c8bbef32f928d9dc89b Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Fri, 8 Sep 2023 08:46:10 +0100 Subject: [PATCH 70/90] Changed command line argument for key-pair --- src/app/test_executive/test_executive.ml | 71 ++++++++++++------------ 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index 0043e188c7f..b2842b83763 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -31,7 +31,7 @@ type inputs = ; archive_image : string option ; debug : bool ; config_path : string option - ; keypairs_path : string + ; keypairs_path : string option ; mock_alias : (string * string) option } @@ -66,43 +66,44 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : "Must provide either --mock command line arg or set MOCK_NETWORK env \ var" ; ignore @@ exit 1 ) ; - let keypairs_path = - if String.(suffix inputs.keypairs_path 1 = "/") then - String.drop_suffix inputs.keypairs_path 1 - else inputs.keypairs_path - in - let keypairs_ls = Stdlib.Sys.readdir keypairs_path in - (* check network-keypairs *) - if - not - ( Array.exists keypairs_ls ~f:(String.equal "network-keypairs") - && (Stdlib.Sys.is_directory @@ keypairs_path ^/ "network-keypairs") ) - then ( - [%log fatal] - "No network-keypairs directory present in %s \n\ - \ Consider cloning the pre-generated keypairs repo: \n\ - \ git clone git@github.com:MinaFoundation/lucy-keypairs.git" - keypairs_path ; - ignore @@ exit 1 ) ; - (* check libp2p-keypairs *) - if - not - ( Array.exists keypairs_ls ~f:(String.equal "libp2p-keypairs") - && (Stdlib.Sys.is_directory @@ keypairs_path ^/ "libp2p-keypairs") ) - then ( - [%log fatal] - "No libp2p-keypairs directory present in %s \n\ - \ Consider cloning the pre-generated keypairs repo: \n\ - \ git clone git@github.com:MinaFoundation/lucy-keypairs.git" - keypairs_path ; - ignore @@ exit 1 ) ; match Inputs.Engine.name with - | "abstract" -> + | "abstract" -> ( if Option.is_none inputs.config_path then ( [%log fatal] "Must provide a config file when using the abstract engine" ; exit 1 ) - else Deferred.unit + else + match inputs.keypairs_path with + | None -> + [%log fatal] + "Must provide a config file when using the abstract engine" ; + exit 1 + | Some path -> + let keypairs_ls = Stdlib.Sys.readdir path in + (* check network-keypairs *) + if + not + ( Array.exists keypairs_ls ~f:(String.equal "network-keypairs") + && (Stdlib.Sys.is_directory @@ path ^/ "network-keypairs") ) + then ( + [%log fatal] + "No network-keypairs directory present in %s \n\ + \ Consider cloning the pre-generated keypairs repo: \n\ + \ git clone git@github.com:MinaFoundation/lucy-keypairs.git" + path ; + exit 1 (* check libp2p-keypairs *) ) + else if + not + ( Array.exists keypairs_ls ~f:(String.equal "libp2p-keypairs") + && (Stdlib.Sys.is_directory @@ path ^/ "libp2p-keypairs") ) + then ( + [%log fatal] + "No libp2p-keypairs directory present in %s \n\ + \ Consider cloning the pre-generated keypairs repo: \n\ + \ git clone git@github.com:MinaFoundation/lucy-keypairs.git" + path ; + exit 1 ) + else Deferred.unit ) | _ -> [%log debug] "Config file is only used for the abstract engine. It will be \ @@ -491,7 +492,7 @@ let keypair_dir_path_arg = let doc = "Path to the pre-generated network and libp2p keypair directory." in let env = Arg.env_var "MINA_KEYPAIRS_PATH" ~doc in Arg.( - required + value & opt (some dir) None & info [ "keypairs-path" ] ~env ~docv:"MINA_KEYPAIRS_PATH" ~doc) @@ -540,7 +541,7 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = path in let set_keypair path = - Engine.Network.keypairs_path := path ; + Option.iter path ~f:(fun p -> Engine.Network.keypairs_path := p) ; path in let set_mina_image image = From c6db7101a38e5e00c889bc261a6d2ca79b220b8b Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Fri, 8 Sep 2023 09:18:54 +0100 Subject: [PATCH 71/90] Revert gcp mina_automation back to what it was for key_pair generation --- src/lib/integration_test_cloud_engine/mina_automation.ml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lib/integration_test_cloud_engine/mina_automation.ml b/src/lib/integration_test_cloud_engine/mina_automation.ml index b2f997f00d2..084dd2f580c 100644 --- a/src/lib/integration_test_cloud_engine/mina_automation.ml +++ b/src/lib/integration_test_cloud_engine/mina_automation.ml @@ -342,6 +342,12 @@ module Network_config = struct { constraints = constraint_constants; genesis = genesis_constants } in (* BLOCK PRODUCER CONFIG *) + let mk_net_keypair keypair_name (pk, sk) = + let keypair = + { Keypair.public_key = Public_key.decompress_exn pk; private_key = sk } + in + Network_keypair.create_network_keypair ~keypair_name ~keypair + in let block_producer_config name keypair = { name; keypair; libp2p_secret = "" } in @@ -417,6 +423,7 @@ module Network_config = struct ; worker_nodes = node.worker_nodes } in + (* NETWORK CONFIG *) { mina_automation_location = cli_inputs.mina_automation_location ; debug_arg = debug From 70fe7f5eac5a0e673870110a9f7d50d3fab4b333 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Fri, 8 Sep 2023 14:27:38 +0100 Subject: [PATCH 72/90] Fix typo --- src/app/test_executive/test_executive.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index b2842b83763..8a29a5f8ed6 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -76,7 +76,7 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : match inputs.keypairs_path with | None -> [%log fatal] - "Must provide a config file when using the abstract engine" ; + "Must provide a keypair file when using the abstract engine" ; exit 1 | Some path -> let keypairs_ls = Stdlib.Sys.readdir path in From d9895acc59d0fe00d5357ad4ab242aafa7ebb3a9 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Fri, 8 Sep 2023 15:20:20 +0100 Subject: [PATCH 73/90] Undo spacing changes --- src/lib/integration_test_lib/dsl.ml | 17 +++++++++++------ src/lib/integration_test_lib/wait_condition.ml | 2 ++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/lib/integration_test_lib/dsl.ml b/src/lib/integration_test_lib/dsl.ml index c7f5bbb707c..149c1abf758 100644 --- a/src/lib/integration_test_lib/dsl.ml +++ b/src/lib/integration_test_lib/dsl.ml @@ -107,11 +107,12 @@ module Make (Engine : Intf.Engine.S) () : let wait_for t condition = let open Wait_condition in - let open Network_time_span in let constants = Engine.Network.constants t.network in - let soft_timeout, hard_timeout = - ( to_span condition.soft_timeout ~constants - , to_span condition.hard_timeout ~constants ) + let soft_timeout = + Network_time_span.to_span condition.soft_timeout ~constants + in + let hard_timeout = + Network_time_span.to_span condition.hard_timeout ~constants in let start_time = Time.now () in [%log' info t.logger] @@ -119,8 +120,12 @@ module Make (Engine : Intf.Engine.S) () : $hard_timeout)" condition.description ~metadata: - [ ("soft_timeout", `String (to_string ~constants condition.soft_timeout)) - ; ("hard_timeout", `String (to_string ~constants condition.hard_timeout)) + [ ( "soft_timeout" + , `String + (Network_time_span.to_string ~constants condition.soft_timeout) ) + ; ( "hard_timeout" + , `String + (Network_time_span.to_string ~constants condition.hard_timeout) ) ] ; let%bind result = match condition.predicate with diff --git a/src/lib/integration_test_lib/wait_condition.ml b/src/lib/integration_test_lib/wait_condition.ml index 37b54994bfc..110b2ee38cf 100644 --- a/src/lib/integration_test_lib/wait_condition.ml +++ b/src/lib/integration_test_lib/wait_condition.ml @@ -70,6 +70,7 @@ struct let wait_condition_id t = t.id let nodes_to_initialize nodes = + let open Network_state in network_state ~id:Nodes_to_initialize ~description: ( nodes |> List.map ~f:Node.id |> String.concat ~sep:", " @@ -84,6 +85,7 @@ struct let node_to_initialize node = nodes_to_initialize [ node ] + (* let blocks_produced ?(active_stake_percentage = 1.0) n = *) let blocks_to_be_produced n = let init state = Predicate_continuation state.blocks_generated in let check init_blocks_generated state = From 61a79608e9fbd058636ac0bcc77d2ca4fdebaf50 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Fri, 8 Sep 2023 21:06:47 +0000 Subject: [PATCH 74/90] (intg test) Remove test config arg from Network_manager.create --- src/app/test_executive/test_executive.ml | 2 +- src/lib/integration_test_abstract_engine/mina_automation.ml | 2 +- src/lib/integration_test_lib/intf.ml | 6 +----- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index 8a29a5f8ed6..b7e7af14d9c 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -372,7 +372,7 @@ let main inputs = in [%log trace] "initializing network manager" ; let%bind net_manager = - Engine.Network_manager.create ~logger network_config T.config + Engine.Network_manager.create ~logger network_config in net_manager_ref := Some net_manager ; [%log trace] "deploying network" ; diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index 82c9286ec89..1a00be85603 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -25,7 +25,7 @@ module Network_manager = struct let run_cmd_or_hard_error t prog args = Util.run_cmd_or_hard_error t.testnet_dir prog args - let create ~logger (network_config : Network_config.t) _test_config = + let create ~logger (network_config : Network_config.t) = let open Malleable_error.Let_syntax in let testnet_dir = network_config.config.config_dir ^/ "testnets" diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index a701976cc01..b50dbd9a441 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -127,11 +127,7 @@ module Engine = struct type t - val create : - logger:Logger.t - -> Network_config.t - -> Test_config.t - -> t Malleable_error.t + val create : logger:Logger.t -> Network_config.t -> t Malleable_error.t val deploy : t -> Network.t Malleable_error.t From 082e3d0cd52da8605894e738181f6d995a39328f Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sat, 9 Sep 2023 04:36:42 +0000 Subject: [PATCH 75/90] (intg test) Add snark coordinator libp2p info --- .../mina_automation.ml | 15 ++++- .../network_config.ml | 64 +++++++++++++++---- src/lib/integration_test_lib/test_config.ml | 11 ++++ 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index 1a00be85603..16341adeaff 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -81,7 +81,7 @@ module Network_manager = struct Out_channel.with_file ~fail_if_exists:true network_config_filename ~f:(fun ch -> Network_config.to_yojson network_config |> Yojson.Safe.to_channel ch ) ; - [%log info] "Writing runtime configuration to %s" genesis_ledger_filename ; + [%log info] "Writing genesis ledger to %s" genesis_ledger_filename ; Out_channel.with_file ~fail_if_exists:true genesis_ledger_filename ~f:(fun ch -> network_config.config.runtime_config |> Yojson.Safe.to_channel ch ) ; @@ -136,6 +136,19 @@ module Network_manager = struct ~data:config.libp2p_peerid ) |> Deferred.return in + let%bind () = + [%log info] "Writing snark coordinator libp2p keypair to %s" + libp2p_kps_base_path ; + Option.iter network_config.config.snark_coordinator_config + ~f:(fun config -> + let keypair_file = + sprintf "%s/%s.json" libp2p_kps_base_path config.name + in + Yojson.Safe.to_file keypair_file config.libp2p_keypair ; + Out_channel.write_all (keypair_file ^ ".peerid") + ~data:config.libp2p_peerid ) + |> Deferred.return + in Malleable_error.return t let deploy t = diff --git a/src/lib/integration_test_abstract_engine/network_config.ml b/src/lib/integration_test_abstract_engine/network_config.ml index 0996253fd9a..baaf54cee49 100644 --- a/src/lib/integration_test_abstract_engine/network_config.ml +++ b/src/lib/integration_test_abstract_engine/network_config.ml @@ -27,7 +27,13 @@ type seed_node_config = [@@deriving to_yojson] type snark_coordinator_config = - { name : string; public_key : string; worker_nodes : int } + { name : string + ; public_key : string + ; worker_nodes : int + ; libp2p_pass : string + ; libp2p_keypair : Yojson.Safe.t + ; libp2p_peerid : string + } [@@deriving to_yojson] type config = @@ -310,16 +316,33 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) cli_inputs.mina_automation_location ^/ "testnets" ^/ network_id ^/ "libp2p_keys" ^/ name ^ ".json" in + let privkey_path name = + let open Core in + cli_inputs.mina_automation_location ^/ "testnets" ^/ network_id + ^/ "block_producer_keys" ^/ name ^ ".json" + in (* instantiate remaining topology information *) let topology = List.map topology ~f:(function | Archive (name, archive_info) -> Test_config.Topology.Archive (name, { archive_info with schema_file; zkapp_table }) - | Node (name, node_info) -> - Node (name, { node_info with libp2p_keyfile = libp2p_keyfile name }) - | x -> - x ) + | Node (name, node_info) -> ( + match node_info.role with + | Test_config.Node_role.Block_producer -> + Node + ( name + , { node_info with + privkey_path = Some (privkey_path name) + ; libp2p_keyfile = libp2p_keyfile name + } ) + | _ -> + Node + (name, { node_info with libp2p_keyfile = libp2p_keyfile name }) + ) + | Snark_coordinator (name, snark_info) -> + Snark_coordinator + (name, { snark_info with libp2p_keyfile = libp2p_keyfile name }) ) in let genesis_keypairs = String.Map.of_alist_exn @@ -331,9 +354,9 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) match snark_coordinator with | None -> None - | Some node -> + | Some snark -> let network_kp = - match String.Map.find genesis_keypairs node.account_name with + match String.Map.find genesis_keypairs snark.account_name with | Some acct -> acct | None -> @@ -342,17 +365,30 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) "Failing because the account key of all initial snark \ coordinators must be in the genesis ledger. name of Node: \ %s. name of Account which does not exist: %s" - node.node_name node.account_name + snark.node_name snark.account_name in failwith failstring in Some - { name = node.node_name - ; public_key = - Public_key.Compressed.to_base58_check - (Public_key.compress network_kp.keypair.public_key) - ; worker_nodes = node.worker_nodes - } + (let[@warning "-8"] (Test_config.Topology.Snark_coordinator + (_, snark_info) ) = + List.find_exn topology ~f:(function + | Snark_coordinator (name, _) -> + String.equal name snark.node_name + | _ -> + false ) + in + { name = snark.node_name + ; public_key = + Public_key.Compressed.to_base58_check + (Public_key.compress network_kp.keypair.public_key) + ; worker_nodes = snark.worker_nodes + ; libp2p_pass = "naughty blue worm" + ; libp2p_keypair = snark_info.libp2p_keypair + ; libp2p_peerid = + Yojson.Safe.to_string snark_info.libp2p_peerid + |> Util.drop_outer_quotes + } ) in (* NETWORK CONFIG *) { debug_arg = debug diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 4af39f95fbc..46417a55cf7 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -144,6 +144,10 @@ module Topology = struct ; docker_image : string ; worker_nodes : int ; snark_worker_fee : string + ; libp2p_pass : string + ; libp2p_keyfile : string + ; libp2p_keypair : Yojson.Safe.t + ; libp2p_peerid : Yojson.Safe.t } [@@deriving to_yojson] @@ -164,6 +168,7 @@ module Topology = struct type node_info = { pk : string ; sk : string + ; privkey_path : string option ; role : Node_role.t ; docker_image : string ; libp2p_pass : string @@ -310,6 +315,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ( node_name , { pk ; sk + ; privkey_path = Some "instantiated in network_config.ml" ; role = Block_producer ; docker_image ; libp2p_pass @@ -333,6 +339,10 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; docker_image ; worker_nodes ; snark_worker_fee = t.snark_worker_fee + ; libp2p_pass + ; libp2p_keyfile = "instantiated in network_config.ml" + ; libp2p_keypair = List.nth_exn libp2p_keypairs num_bp + ; libp2p_peerid = `String List.(nth_exn libp2p_peerids num_bp) } ) in let snark_coordinator = @@ -365,6 +375,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ( node_name , { pk ; sk + ; privkey_path = None ; role = Seed_node ; docker_image ; libp2p_pass From 54708ac77fbdf7ebb7fedc86933b6f70045d192b Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sat, 9 Sep 2023 19:27:04 +0000 Subject: [PATCH 76/90] (intg test) Undo function renaming --- .../abstract_network.ml | 12 ++++----- src/lib/integration_test_lib/intf.ml | 26 +++++++++++++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index 59e0f0effa2..8dc7fd7e73a 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -386,7 +386,7 @@ module Node = struct let%bind () = after (Time.Span.of_sec initial_delay_sec) in retry num_tries - let get_peer_ids ~logger t = + let get_peer_id ~logger t = let open Deferred.Or_error.Let_syntax in [%log info] "Getting node's peer_id, and the peer_ids of node's peers" ~metadata:(logger_metadata t) ; @@ -394,7 +394,7 @@ module Node = struct let%bind query_result_obj = exec_graphql_request ~logger ~node:t ~query_name:"query_peer_id" query_obj in - [%log info] "get_peer_ids, finished exec_graphql_request" ; + [%log info] "get_peer_id, finished exec_graphql_request" ; let self_id_obj = query_result_obj.daemonStatus.addrsAndPorts.peer in let%bind self_id = match self_id_obj with @@ -405,13 +405,13 @@ module Node = struct in let peers = query_result_obj.daemonStatus.peers |> Array.to_list in let peer_ids = List.map peers ~f:(fun peer -> peer.peerId) in - [%log info] - "get_peer_ids, result of graphql query (self_id,[peers]) (%s,%s)" self_id + [%log info] "get_peer_id, result of graphql query (self_id,[peers]) (%s,%s)" + self_id (String.concat ~sep:" " peer_ids) ; return (self_id, peer_ids) - let must_get_peer_ids ~logger t = - get_peer_ids ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error + let must_get_peer_id ~logger t = + get_peer_id ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error let get_best_chain ?max_length ~logger t = let open Deferred.Or_error.Let_syntax in diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index b50dbd9a441..5c0c7d5cf5f 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -60,7 +60,33 @@ module Engine = struct *) val should_be_running : t -> bool +<<<<<<< HEAD val get_ingress_uri : t -> Uri.t +======= + val get_pooled_zkapp_commands : + logger:Logger.t + -> t + -> pk:Signature_lib.Public_key.Compressed.t + -> string list Deferred.Or_error.t + + val get_peer_id : + logger:Logger.t -> t -> (string * string list) Deferred.Or_error.t + + val must_get_peer_id : + logger:Logger.t -> t -> (string * string list) Malleable_error.t + + val get_best_chain : + ?max_length:int + -> logger:Logger.t + -> t + -> best_chain_block list Deferred.Or_error.t + + val must_get_best_chain : + ?max_length:int + -> logger:Logger.t + -> t + -> best_chain_block list Malleable_error.t +>>>>>>> 6131f41e37 ((intg test) Undo function renaming) val dump_archive_data : logger:Logger.t -> t -> data_file:string -> unit Malleable_error.t From adb795b05ace09d4b2eea0126780fd5da0d5133b Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sun, 10 Sep 2023 04:26:59 +0000 Subject: [PATCH 77/90] (intg test) Add git build field for commit/tag in topology --- src/lib/integration_test_lib/test_config.ml | 89 ++++++++++++++++----- 1 file changed, 68 insertions(+), 21 deletions(-) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 46417a55cf7..3cc199b3528 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -72,18 +72,43 @@ module Epoch_data = struct type t = { staking : Data.t; next : Data.t option } end +module Git_build = struct + type t = Commit of string | Tag of string [@@deriving to_yojson] + + let to_yojson : t -> Yojson.Safe.t = function + | Commit commit -> + `Assoc [ ("commit", `String commit) ] + | Tag tag -> + `Assoc [ ("tag", `String tag) ] +end + module Archive_node = struct - type t = { node_name : string; account_name : string; docker_image : string } + type t = + { node_name : string + ; account_name : string + ; docker_image : string option + ; git_build : Git_build.t option + } [@@deriving to_yojson] end module Block_producer_node = struct - type t = { node_name : string; account_name : string; docker_image : string } + type t = + { node_name : string + ; account_name : string + ; docker_image : string option + ; git_build : Git_build.t option + } [@@deriving to_yojson] end module Seed_node = struct - type t = { node_name : string; account_name : string; docker_image : string } + type t = + { node_name : string + ; account_name : string + ; docker_image : string option + ; git_build : Git_build.t option + } [@@deriving to_yojson] end @@ -91,7 +116,8 @@ module Snark_coordinator_node = struct type t = { node_name : string ; account_name : string - ; docker_image : string + ; docker_image : string option + ; git_build : Git_build.t option ; worker_nodes : int } [@@deriving to_yojson] @@ -141,7 +167,8 @@ module Topology = struct { pk : string ; sk : string ; role : Node_role.t - ; docker_image : string + ; docker_image : string option + ; git_build : Git_build.t option ; worker_nodes : int ; snark_worker_fee : string ; libp2p_pass : string @@ -152,14 +179,19 @@ module Topology = struct [@@deriving to_yojson] type snark_worker_info = - { pk : string; sk : string; role : Node_role.t; docker_image : string } + { pk : string + ; sk : string + ; role : Node_role.t + ; docker_image : string option + } [@@deriving to_yojson] type archive_info = { pk : string ; sk : string ; role : Node_role.t - ; docker_image : string + ; docker_image : string option + ; git_build : Git_build.t option ; schema_file : string ; zkapp_table : string } @@ -170,7 +202,8 @@ module Topology = struct ; sk : string ; privkey_path : string option ; role : Node_role.t - ; docker_image : string + ; docker_image : string option + ; git_build : Git_build.t option ; libp2p_pass : string ; libp2p_keyfile : string ; libp2p_keypair : Yojson.Safe.t @@ -219,7 +252,13 @@ let default = ; epoch_data = None ; archive_nodes = [] ; block_producers = [] - ; seed_nodes = [] + ; seed_nodes = + [ { node_name = "default-seed" + ; account_name = "default-seed-key" + ; docker_image = None + ; git_build = None + } + ] ; snark_coordinator = None ; snark_worker_fee = "0.025" ; log_precomputed_blocks = false @@ -308,7 +347,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : in let libp2p_pass = "naughty blue worm" in let topology_of_block_producer n - { Block_producer_node.account_name; node_name; docker_image } : + { Block_producer_node.account_name; node_name; docker_image; git_build } : Topology.top_info = let pk, sk = pk_sk account_name n in Node @@ -318,6 +357,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; privkey_path = Some "instantiated in network_config.ml" ; role = Block_producer ; docker_image + ; git_build ; libp2p_pass ; libp2p_keyfile = "instantiated in network_config.ml" ; libp2p_keypair = List.nth_exn libp2p_keypairs n @@ -328,6 +368,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : { Snark_coordinator_node.account_name ; node_name ; docker_image + ; git_build ; worker_nodes } : Topology.top_info = let pk, sk = pk_sk account_name num_bp in @@ -337,6 +378,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; sk ; role = Snark_coordinator ; docker_image + ; git_build ; worker_nodes ; snark_worker_fee = t.snark_worker_fee ; libp2p_pass @@ -353,8 +395,8 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : [ sc ] in let topology_of_archive n - { Archive_node.account_name; node_name; docker_image } : Topology.top_info - = + { Archive_node.account_name; node_name; docker_image; git_build } : + Topology.top_info = let n = n + num_bp_sc in let pk, sk = pk_sk account_name n in Archive @@ -363,11 +405,13 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; sk ; role = Archive_node ; docker_image + ; git_build ; schema_file = "instantiated in network_config.ml" ; zkapp_table = "instantiated in network_config.ml" } ) in - let topology_of_seed n { Seed_node.account_name; node_name; docker_image } : + let topology_of_seed n + { Seed_node.account_name; node_name; docker_image; git_build } : Topology.top_info = let n = n + num_bp_sc_an in let pk, sk = pk_sk account_name n in @@ -378,6 +422,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; privkey_path = None ; role = Seed_node ; docker_image + ; git_build ; libp2p_pass ; libp2p_keyfile = "instantiated in network_config.ml" ; libp2p_keypair = List.nth_exn libp2p_keypairs n @@ -393,9 +438,6 @@ let test_account ?(pk = "") ?(timing = Mina_base.Account.Timing.Untimed) account_name balance : Test_Account.t = { account_name; balance; timing; pk } -let bp node_name ?(account_name = node_name ^ "-key") docker_image = - { Block_producer_node.node_name; account_name; docker_image } - module Unit_tests = struct let test_config = { default with @@ -409,28 +451,33 @@ module Unit_tests = struct ; block_producers = [ { node_name = "receiver" ; account_name = "receiver-key" - ; docker_image = "bp-image" + ; docker_image = Some "bp-image" + ; git_build = None } ; { node_name = "empty-node-0" ; account_name = "empty-bp-key" - ; docker_image = "bp-image-0" + ; docker_image = None + ; git_build = Some (Commit "6131f41e") } ; { node_name = "empty-node-1" ; account_name = "empty-bp-key" - ; docker_image = "bp-image-1" + ; docker_image = Some "bp-image-1" + ; git_build = None } ] ; snark_coordinator = Some { node_name = "snark-node" ; account_name = "snark-node-key" - ; docker_image = "snark-coordinator-image" + ; docker_image = Some "snark-coordinator-image" + ; git_build = None ; worker_nodes = 4 } ; seed_nodes = [ { node_name = "seed-node-0" ; account_name = "seed-node-0-key" - ; docker_image = "seed-image-0" + ; docker_image = Some "seed-image-0" + ; git_build = None } ] ; txpool_max_size = 10_000_000 From f36698b754c754a56741456a1e93cc0312e26b0f Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sun, 10 Sep 2023 04:29:08 +0000 Subject: [PATCH 78/90] (intg test) Add node config helpers with default values --- src/app/test_executive/test_common.ml | 46 +++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/app/test_executive/test_common.ml b/src/app/test_executive/test_common.ml index 15c60e432e4..52ab99c253d 100644 --- a/src/app/test_executive/test_common.ml +++ b/src/app/test_executive/test_common.ml @@ -449,4 +449,50 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ; vesting_period = Mina_numbers.Global_slot_span.of_int vesting_period ; vesting_increment = Amount.of_nanomina_int_exn vesting_increment } + + module Node_config = struct + open Inputs.Engine.Network + + let archive node_name ?(account_name = node_name ^ "-key") + ?(docker_image = !archive_image) ?git_build () = + let docker_image = + if Option.is_some git_build then None else docker_image + in + { Test_config.Archive_node.node_name + ; account_name + ; docker_image + ; git_build + } + + let bp node_name ?(account_name = node_name ^ "-key") + ?(docker_image = !mina_image) ?git_build () = + let docker_image = + if Option.is_some git_build then None else Some docker_image + in + { Test_config.Block_producer_node.node_name + ; account_name + ; docker_image + ; git_build + } + + let seed node_name ?(account_name = node_name ^ "-key") + ?(docker_image = !mina_image) ?git_build () = + let docker_image = + if Option.is_some git_build then None else Some docker_image + in + { Test_config.Seed_node.node_name; account_name; docker_image; git_build } + + let snark node_name ?(account_name = node_name ^ "-key") + ?(docker_image = !mina_image) ?git_build worker_nodes = + let docker_image = + if Option.is_some git_build then None else Some docker_image + in + Some + { Test_config.Snark_coordinator_node.node_name + ; account_name + ; docker_image + ; git_build + ; worker_nodes + } + end end From fa82138b71aa995ae1aee754f34ac8f2910d96eb Mon Sep 17 00:00:00 2001 From: Quantifier Date: Sun, 10 Sep 2023 04:30:20 +0000 Subject: [PATCH 79/90] (intg test) Update tests with helpers --- .../block_production_priority.ml | 21 ++++------ src/app/test_executive/block_reward_test.ml | 3 +- .../test_executive/chain_reliability_test.ml | 7 +--- src/app/test_executive/gossip_consistency.ml | 4 +- src/app/test_executive/medium_bootstrap.ml | 7 +--- src/app/test_executive/mock.ml | 39 +++++-------------- src/app/test_executive/payments_test.ml | 26 +++++++------ .../test_executive/peers_reliability_test.ml | 7 +--- src/app/test_executive/snarkyjs.ml | 3 +- .../test_executive/verification_key_update.ml | 13 ++----- src/app/test_executive/zkapps.ml | 13 ++----- src/app/test_executive/zkapps_nonce_test.ml | 13 ++----- src/app/test_executive/zkapps_timing.ml | 8 +--- 13 files changed, 54 insertions(+), 110 deletions(-) diff --git a/src/app/test_executive/block_production_priority.ml b/src/app/test_executive/block_production_priority.ml index fb50ef450e4..dabb703ee5d 100644 --- a/src/app/test_executive/block_production_priority.ml +++ b/src/app/test_executive/block_production_priority.ml @@ -14,6 +14,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in + let open Node_config in { default with genesis_ledger = [ test_account "receiver-key" "9999999" @@ -24,20 +25,14 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let i_str = Int.to_string i in test_account ("sender-account-" ^ i_str) "10000" ) ; block_producers = - [ bp "receiver" !Network.mina_image - ; bp "empty_node-1" ~account_name:"empty-bp-key" !Network.mina_image - ; bp "empty_node-2" ~account_name:"empty-bp-key" !Network.mina_image - ; bp "empty_node-3" ~account_name:"empty-bp-key" !Network.mina_image - ; bp "empty_node-4" ~account_name:"empty-bp-key" !Network.mina_image - ; bp "observer" ~account_name:"empty-bp-key" !Network.mina_image + [ bp "receiver" () + ; bp "empty_node-1" ~account_name:"empty-bp-key" () + ; bp "empty_node-2" ~account_name:"empty-bp-key" () + ; bp "empty_node-3" ~account_name:"empty-bp-key" () + ; bp "empty_node-4" ~account_name:"empty-bp-key" () + ; bp "observer" ~account_name:"empty-bp-key" () ] - ; snark_coordinator = - Some - { node_name = "snark-node" - ; account_name = "snark-node-key" - ; docker_image = !Network.mina_image - ; worker_nodes = 4 - } + ; snark_coordinator = snark "snark-node" 4 ; txpool_max_size = 10_000_000 ; snark_worker_fee = "0.0001" ; proof_config = diff --git a/src/app/test_executive/block_reward_test.ml b/src/app/test_executive/block_reward_test.ml index cf5275e5d59..2e58ed81332 100644 --- a/src/app/test_executive/block_reward_test.ml +++ b/src/app/test_executive/block_reward_test.ml @@ -13,9 +13,10 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in + let open Node_config in { default with genesis_ledger = [ test_account "node-key" "1000" ] - ; block_producers = [ bp "node" !Network.mina_image ] + ; block_producers = [ bp "node" () ] } let run network t = diff --git a/src/app/test_executive/chain_reliability_test.ml b/src/app/test_executive/chain_reliability_test.ml index cbb63949554..ab20ab79397 100644 --- a/src/app/test_executive/chain_reliability_test.ml +++ b/src/app/test_executive/chain_reliability_test.ml @@ -12,17 +12,14 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in + let open Node_config in { default with genesis_ledger = [ test_account "node-a-key" "1000" ; test_account "node-b-key" "1000" ; test_account "node-c-key" "0" ] - ; block_producers = - [ bp "node-a" !Network.mina_image - ; bp "node-b" !Network.mina_image - ; bp "node-c" !Network.mina_image - ] + ; block_producers = [ bp "node-a" (); bp "node-b" (); bp "node-c" () ] } let run network t = diff --git a/src/app/test_executive/gossip_consistency.ml b/src/app/test_executive/gossip_consistency.ml index 9272ccb068f..db895944121 100644 --- a/src/app/test_executive/gossip_consistency.ml +++ b/src/app/test_executive/gossip_consistency.ml @@ -12,11 +12,11 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in + let open Node_config in { default with genesis_ledger = [ test_account "node-a-key" "1000"; test_account "node-b-key" "1000" ] - ; block_producers = - [ bp "node-a" !Network.mina_image; bp "node-b" !Network.mina_image ] + ; block_producers = [ bp "node-a" (); bp "node-b" () ] } let run network t = diff --git a/src/app/test_executive/medium_bootstrap.ml b/src/app/test_executive/medium_bootstrap.ml index 9b8925c4932..e092c2d9517 100644 --- a/src/app/test_executive/medium_bootstrap.ml +++ b/src/app/test_executive/medium_bootstrap.ml @@ -13,6 +13,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in + let open Node_config in { default with k = 2 ; genesis_ledger = @@ -20,11 +21,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ; test_account "node-b-key" "1000" ; test_account "node-c-key" "0" ] - ; block_producers = - [ bp "node-a" !Network.mina_image - ; bp "node-b" !Network.mina_image - ; bp "node-c" !Network.mina_image - ] + ; block_producers = [ bp "node-a" (); bp "node-b" (); bp "node-c" () ] } (* diff --git a/src/app/test_executive/mock.ml b/src/app/test_executive/mock.ml index 920b2024405..9db9d75d21b 100644 --- a/src/app/test_executive/mock.ml +++ b/src/app/test_executive/mock.ml @@ -2,8 +2,6 @@ open Core open Integration_test_lib module Make (Inputs : Intf.Test.Inputs_intf) = struct - open Inputs.Engine - open Test_common.Make (Inputs) let num_extra_keys = 1000 @@ -12,6 +10,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in + let open Node_config in { default with genesis_ledger = [ test_account "receiver-key" "9999999" @@ -22,18 +21,13 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let i_str = Int.to_string i in test_account ("sender-account-" ^ i_str) "10000" ) ; block_producers = - [ bp "receiver" "receiver-docker-image" - ; bp "empty_node-1" ~account_name:"empty-bp-key" !Network.mina_image - ; bp "empty_node-2" ~account_name:"empty-bp-key" !Network.mina_image - ; bp "observer" ~account_name:"empty-bp-key" "observer-docker-image" + [ bp "receiver" ~docker_image:"receiver-docker-image" () + ; bp "empty_node-1" ~account_name:"empty-bp-key" () + ; bp "empty_node-2" ~account_name:"empty-bp-key" () + ; bp "observer" ~account_name:"empty-bp-key" + ~git_build:(Commit "abcdef0123") () ] - ; snark_coordinator = - Some - { node_name = "snark-node" - ; account_name = "snark-node-key" - ; docker_image = !Network.mina_image - ; worker_nodes = 4 - } + ; snark_coordinator = snark "snark-node" 4 ; txpool_max_size = 10_000_000 ; snark_worker_fee = "0.0001" ; proof_config = @@ -43,23 +37,8 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct Some Runtime_config.Proof_keys.Transaction_capacity.small } ; seed_nodes = - [ { node_name = "seed-0" - ; account_name = "seed-0-key" - ; docker_image = "seed-docker-image" - } - ; { node_name = "seed-1" - ; account_name = "seed-1-key" - ; docker_image = !Network.mina_image - } - ] - ; archive_nodes = - [ { node_name = "archive-node" - ; account_name = "archive-node-key" - ; docker_image = - Option.value !Network.archive_image - ~default:"default-archive-image" - } - ] + [ seed "seed-0" ~docker_image:"seed-docker-image" (); seed "seed-1" () ] + ; archive_nodes = [ archive "archive-node" () ] } let run _network _t = Malleable_error.return () diff --git a/src/app/test_executive/payments_test.ml b/src/app/test_executive/payments_test.ml index bd8dddeb472..a8a38645db9 100644 --- a/src/app/test_executive/payments_test.ml +++ b/src/app/test_executive/payments_test.ml @@ -16,6 +16,18 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct (* TODO: test account creation fee *) let config = let open Test_config in + let open Node_config in + let make_timing ~min_balance ~cliff_time ~cliff_amount ~vesting_period + ~vesting_increment : Mina_base.Account_timing.t = + let open Currency in + Timed + { initial_minimum_balance = Balance.of_nanomina_int_exn min_balance + ; cliff_time = Mina_numbers.Global_slot_since_genesis.of_int cliff_time + ; cliff_amount = Amount.of_nanomina_int_exn cliff_amount + ; vesting_period = Mina_numbers.Global_slot_span.of_int vesting_period + ; vesting_increment = Amount.of_nanomina_int_exn vesting_increment + } + in { default with genesis_ledger = [ test_account "untimed-node-a-key" "400000" (* 400_000_000_000_000 *) @@ -32,19 +44,9 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ; test_account "fish2" "100" ] ; block_producers = - [ bp "untimed-node-a" !Network.mina_image - ; bp "untimed-node-b" !Network.mina_image - ; bp "timed-node-c" !Network.mina_image - ] - ; snark_coordinator = - Some - { node_name = "snark-node" - ; account_name = "snark-node-key1" - ; docker_image = !Network.mina_image - ; worker_nodes = 4 - } + [ bp "untimed-node-a" (); bp "untimed-node-b" (); bp "timed-node-c" () ] + ; snark_coordinator = snark "snark-node" 4 ; snark_worker_fee = "0.0001" - ; archive_nodes = [] ; proof_config = { proof_config_default with work_delay = Some 1 diff --git a/src/app/test_executive/peers_reliability_test.ml b/src/app/test_executive/peers_reliability_test.ml index 39d91e81c84..948cdc3584c 100644 --- a/src/app/test_executive/peers_reliability_test.ml +++ b/src/app/test_executive/peers_reliability_test.ml @@ -13,17 +13,14 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in + let open Node_config in { default with genesis_ledger = [ test_account "node-a-key" "700000" ; test_account "node-b-key" "700000" ; test_account "node-c-key" "800000" ] - ; block_producers = - [ bp "node-a" !Network.mina_image - ; bp "node-b" !Network.mina_image - ; bp "node-c" !Network.mina_image - ] + ; block_producers = [ bp "node-a" (); bp "node-b" (); bp "node-c" () ] } let run network t = diff --git a/src/app/test_executive/snarkyjs.ml b/src/app/test_executive/snarkyjs.ml index 63c148542f0..5b42e4f843c 100644 --- a/src/app/test_executive/snarkyjs.ml +++ b/src/app/test_executive/snarkyjs.ml @@ -13,10 +13,11 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in + let open Node_config in { default with genesis_ledger = [ test_account "node-key" "8000000"; test_account "extra-key" "10" ] - ; block_producers = [ bp "node" !Network.mina_image ] + ; block_producers = [ bp "node" () ] } let check_and_print_stout_stderr ~logger process = diff --git a/src/app/test_executive/verification_key_update.ml b/src/app/test_executive/verification_key_update.ml index 981fcd8ce15..8a890197963 100644 --- a/src/app/test_executive/verification_key_update.ml +++ b/src/app/test_executive/verification_key_update.ml @@ -65,22 +65,15 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in + let open Node_config in { default with genesis_ledger = [ test_account "whale1-key" "9000000000" ; test_account "whale2-key" "1000000000" ; test_account "snark-node-key" "100" ] - ; block_producers = - [ bp "whale1" !Network.mina_image; bp "whale2" !Network.mina_image ] - ; archive_nodes = [] - ; snark_coordinator = - Some - { node_name = "snark-node" - ; account_name = "snark-node-key" - ; docker_image = !Network.mina_image - ; worker_nodes = 2 - } + ; block_producers = [ bp "whale1" (); bp "whale2" () ] + ; snark_coordinator = snark "snark-node" 2 ; snark_worker_fee = "0.0001" } diff --git a/src/app/test_executive/zkapps.ml b/src/app/test_executive/zkapps.ml index ba6db05a9bd..510945f3f59 100644 --- a/src/app/test_executive/zkapps.ml +++ b/src/app/test_executive/zkapps.ml @@ -14,6 +14,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in + let open Node_config in { default with genesis_ledger = [ test_account "node-a-key" "8000000000" @@ -22,16 +23,8 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ; test_account "fish2" "3000" ; test_account "snark-node-key" "0" ] - ; block_producers = - [ bp "node-a" !Network.mina_image; bp "node-b" !Network.mina_image ] - ; archive_nodes = [] - ; snark_coordinator = - Some - { node_name = "snark-node" - ; account_name = "snark-node-key" - ; docker_image = !Network.mina_image - ; worker_nodes = 2 - } + ; block_producers = [ bp "node-a" (); bp "node-b" () ] + ; snark_coordinator = snark "snark-node" 2 ; snark_worker_fee = "0.0001" ; proof_config = { proof_config_default with diff --git a/src/app/test_executive/zkapps_nonce_test.ml b/src/app/test_executive/zkapps_nonce_test.ml index c175b648278..3a83b2090fe 100644 --- a/src/app/test_executive/zkapps_nonce_test.ml +++ b/src/app/test_executive/zkapps_nonce_test.ml @@ -14,6 +14,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in + let open Node_config in { default with genesis_ledger = [ test_account "node-a-key" "8000000000" @@ -22,16 +23,8 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ; test_account "fish2" "3000" ; test_account "snark-node-key" "0" ] - ; block_producers = - [ bp "node-a" !Network.mina_image; bp "node-b" !Network.mina_image ] - ; archive_nodes = [] - ; snark_coordinator = - Some - { node_name = "snark-node" - ; account_name = "snark-node-key" - ; docker_image = !Network.mina_image - ; worker_nodes = 5 - } + ; block_producers = [ bp "node-a" (); bp "node-b" () ] + ; snark_coordinator = snark "snark-node" 5 ; snark_worker_fee = "0.0001" ; proof_config = { proof_config_default with diff --git a/src/app/test_executive/zkapps_timing.ml b/src/app/test_executive/zkapps_timing.ml index f513a60a818..c42623b6951 100644 --- a/src/app/test_executive/zkapps_timing.ml +++ b/src/app/test_executive/zkapps_timing.ml @@ -13,18 +13,14 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let config = let open Test_config in + let open Node_config in { default with genesis_ledger = [ test_account "node-a-key" "8000000000" ; test_account "node-b-key" "1000000000" ; test_account "node-c-key" "1000000000" ] - ; block_producers = - [ bp "node-a" !Network.mina_image - ; bp "node-b" !Network.mina_image - ; bp "node-c" !Network.mina_image - ] - ; archive_nodes = [] + ; block_producers = [ bp "node-a" (); bp "node-b" (); bp "node-c" () ] } let run network t = From 37a6e6462c152b13872bdc1129d668e475aa9b98 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Mon, 11 Sep 2023 19:54:54 +0000 Subject: [PATCH 80/90] (intg test) Add --start-slot-since-genesis to node run replayer command --- integration_tests/mock_config.json | 5 +++-- .../integration_test_abstract_engine/test_data/config.json | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/integration_tests/mock_config.json b/integration_tests/mock_config.json index 9dfa9b52c14..c048100e342 100644 --- a/integration_tests/mock_config.json +++ b/integration_tests/mock_config.json @@ -69,9 +69,10 @@ "name": "run_replayer", "args": { "network_id": "string", - "node_id": "string" + "node_id": "string", + "start_slot_since_genesis": "int" }, - "command": "MOCK_NETWORK node run-replayer --network-id {{network_id}} --node-id {{node_id}}" + "command": "MOCK_NETWORK node run-replayer --network-id {{network_id}} --node-id {{node_id}} --start-slot-since-genesis {{start_slot_since_genesis}}" } ] } diff --git a/src/lib/integration_test_abstract_engine/test_data/config.json b/src/lib/integration_test_abstract_engine/test_data/config.json index 4c33235bf99..8038b52fb55 100644 --- a/src/lib/integration_test_abstract_engine/test_data/config.json +++ b/src/lib/integration_test_abstract_engine/test_data/config.json @@ -72,9 +72,10 @@ "name": "run_replayer", "args": { "network_id":"string", - "node_id":"string" + "node_id":"string", + "start_slot_since_genesis": "int" }, - "command": "minimina node run-replayer --network-id {{network_id}} --node-id {{node_id}}" + "command": "minimina node run-replayer --network-id {{network_id}} --node-id {{node_id}} --start-slot-since-genesis {{start_slot_since_genesis}}" }, { "name": "echo", From 2487802117111523ce587da8300797e3ec84d025 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Mon, 11 Sep 2023 19:57:50 +0000 Subject: [PATCH 81/90] (intg test) Generate unique genesis accounts --- .../abstract_network.ml | 5 +++- src/lib/integration_test_lib/util.ml | 25 ++++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index 8dc7fd7e73a..02fb1771ac3 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -1104,7 +1104,10 @@ module Node = struct let run_replayer ~logger (t : t) = [%log info] "Running replayer on archived data node: %s" t.node_id ; let args = - [ ("network_id", `String t.network_id); ("node_id", `String t.node_id) ] + [ ("network_id", `String t.network_id) + ; ("node_id", `String t.node_id) + ; ("start_slot_since_genesis", `Int start_slot_since_genesis) + ] in try let%bind replay = diff --git a/src/lib/integration_test_lib/util.ml b/src/lib/integration_test_lib/util.ml index c24faa9489c..2f8e075b3d5 100644 --- a/src/lib/integration_test_lib/util.ml +++ b/src/lib/integration_test_lib/util.ml @@ -23,15 +23,28 @@ let drop_outer_quotes s = let s = drop_suffix s n in s +(** Pulls [num_keypairs] unique keypairs from [path] *) let pull_keypairs path num_keypairs = + let random = + Random.State.make_self_init () |> Splittable_random.State.create + in + (* elem must be in list *) + let drop_elem list elem = + let open List in + let idx = findi list ~f:(fun _ x -> x = elem) |> Option.value_exn |> fst in + let left, right = split_n list idx in + match right with [] -> left | _ :: tl -> left @ tl + in let random_nums = - let random = - Random.State.make_self_init () |> Splittable_random.State.create + let rec aux rem acc n = + let open Quickcheck.Generator in + if n = 0 then acc + else + let next = of_list rem |> generate ~size:num_keypairs ~random in + let rem = drop_elem rem next in + aux rem (next :: acc) (n - 1) in - let open Quickcheck.Generator in - of_list (List.range ~stop:`inclusive 1 10_000) - |> list_with_length num_keypairs - |> generate ~size:num_keypairs ~random + aux (List.range ~stop:`inclusive 1 10_000) [] num_keypairs in (* network keypairs + private keys *) let keypair_name = sprintf "network-keypairs/network-keypair-%d" in From 048bc995dc524bebd6ed9c1fa8c4620fb480a923 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 12 Sep 2023 18:05:50 +0000 Subject: [PATCH 82/90] (intg test) Replace schema_file and zkapp_table fields for schema_files --- integration_tests/return_spec.json | 2 +- src/lib/integration_test_abstract_engine/network_config.ml | 4 +++- src/lib/integration_test_lib/test_config.ml | 6 ++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integration_tests/return_spec.json b/integration_tests/return_spec.json index a6a5d8d4a6d..96d889c4f8c 100644 --- a/integration_tests/return_spec.json +++ b/integration_tests/return_spec.json @@ -35,7 +35,7 @@ "network_id": "string", "node_id": "string" }, - "dump_precomputed_block": { + "dump_precomputed_blocks": { "blocks": "string", "network_id": "string", "node_id": "string" diff --git a/src/lib/integration_test_abstract_engine/network_config.ml b/src/lib/integration_test_abstract_engine/network_config.ml index baaf54cee49..f005479ed63 100644 --- a/src/lib/integration_test_abstract_engine/network_config.ml +++ b/src/lib/integration_test_abstract_engine/network_config.ml @@ -326,7 +326,9 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) List.map topology ~f:(function | Archive (name, archive_info) -> Test_config.Topology.Archive - (name, { archive_info with schema_file; zkapp_table }) + ( name + , { archive_info with schema_files = mina_archive_schema_aux_files } + ) | Node (name, node_info) -> ( match node_info.role with | Test_config.Node_role.Block_producer -> diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 3cc199b3528..7c70d1017a2 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -192,8 +192,7 @@ module Topology = struct ; role : Node_role.t ; docker_image : string option ; git_build : Git_build.t option - ; schema_file : string - ; zkapp_table : string + ; schema_files : string list } [@@deriving to_yojson] @@ -406,8 +405,7 @@ let topology_of_test_config t private_keys libp2p_keypairs libp2p_peerids : ; role = Archive_node ; docker_image ; git_build - ; schema_file = "instantiated in network_config.ml" - ; zkapp_table = "instantiated in network_config.ml" + ; schema_files = [] } ) in let topology_of_seed n From 60a21f1bcde64d051eb8a9fd6662dd77950987df Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 12 Sep 2023 21:19:42 +0000 Subject: [PATCH 83/90] (intg test) Rename network runner arg --- src/app/test_executive/test_executive.ml | 36 ++++++++++--------- .../abstract_network.ml | 2 +- .../config_util.ml | 13 +++---- .../kubernetes_network.ml | 2 +- .../mina_automation.ml | 2 +- .../terraform.ml | 6 ++-- src/lib/integration_test_lib/intf.ml | 4 +-- 7 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index b7e7af14d9c..f64e0171805 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -32,7 +32,7 @@ type inputs = ; debug : bool ; config_path : string option ; keypairs_path : string option - ; mock_alias : (string * string) option + ; network_runner_alias : (string * string) option } let test_name (test : test) @@ -58,13 +58,13 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : inputs.test_inputs in if - String.(test_name inputs.test (module Inputs) = "mock") - && Option.is_none inputs.mock_alias - && (Option.is_none @@ Sys.getenv "MOCK_NETWORK") + String.(Inputs.Engine.name = "abstract") + && Option.is_none inputs.network_runner_alias + && (Option.is_none @@ Sys.getenv "MINA_NETWORK_RUNNER") then ( [%log fatal] - "Must provide either --mock command line arg or set MOCK_NETWORK env \ - var" ; + "Must provide either --alias command line arg or set the \ + MINA_NETWORK_RUNNER env var to the path of your network runner binary" ; ignore @@ exit 1 ) ; match Inputs.Engine.name with | "abstract" -> ( @@ -79,6 +79,8 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : "Must provide a keypair file when using the abstract engine" ; exit 1 | Some path -> + [%log info] "Using network runner: %s" + (Option.value_exn inputs.network_runner_alias |> snd) ; let keypairs_ls = Stdlib.Sys.readdir path in (* check network-keypairs *) if @@ -407,7 +409,6 @@ let main inputs = else ( [%log info] "starting the daemons within the pods" ; let start_print (node : Engine.Network.Node.t) = - let open Malleable_error.Let_syntax in [%log info] "starting %s..." (Engine.Network.Node.id node) ; let%bind res = Engine.Network.Node.start ~fresh_state:false node in [%log info] "%s started" (Engine.Network.Node.id node) ; @@ -439,10 +440,11 @@ let main inputs = let%bind () = Dsl.(wait_for dsl @@ Wait_condition.nodes_to_initialize seed_nodes) in + [%log info] "all seed nodes started" ; let%bind () = Malleable_error.List.iter non_seed_pods ~f:start_print in - [%log info] "daemons started" ; + [%log info] "all nodes started" ; [%log trace] "executing test" ; T.run network dsl ) ) in @@ -480,13 +482,15 @@ let config_path_arg = & opt (some non_dir_file) None & info [ "config-path"; "config" ] ~env ~docv:"MINA_CI_CONFIG_PATH" ~doc) -let mock_alias_arg = - let doc = "Alias to use for the mock network binary." in - let env = Arg.env_var "MOCK_NETWORK" ~doc in +let network_runner_alias_arg = + let doc = "Alias for the network runner binary." in + let env = Arg.env_var "MINA_NETWORK_RUNNER" ~doc in Arg.( value & opt (some string) None - & info [ "mock-network"; "mock"; "alias" ] ~env ~docv:"MOCK_NETWORK" ~doc) + & info + [ "network-runner-alias"; "network-runner"; "alias" ] + ~env ~docv:"MINA_NETWORK_RUNNER" ~doc) let keypair_dir_path_arg = let doc = "Path to the pre-generated network and libp2p keypair directory." in @@ -554,7 +558,7 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = in let set_alias alias = let alias = Option.map alias ~f:(fun a -> ("MOCK_NETWORK", a)) in - Engine.Network.alias := alias ; + Engine.Network.network_runner_alias := alias ; alias in let test_inputs_with_cli_inputs_arg = @@ -565,7 +569,7 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = in let inputs_term = let cons_inputs test_inputs test debug archive_image mina_image config_path - keypairs_path mock_alias = + keypairs_path network_runner_alias = { test_inputs ; test ; mina_image @@ -573,7 +577,7 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = ; debug ; config_path ; keypairs_path - ; mock_alias + ; network_runner_alias } in Term.( @@ -584,7 +588,7 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = $ (const set_mina_image $ mina_image_arg) $ (const set_config $ config_path_arg) $ (const set_keypair $ keypair_dir_path_arg) - $ (const set_alias $ mock_alias_arg)) + $ (const set_alias $ network_runner_alias_arg)) in (Term.(const start $ inputs_term), info) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index 02fb1771ac3..d1841aec210 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -13,7 +13,7 @@ let keypairs_path = keypairs_path let mina_image = mina_image -let alias = alias +let network_runner_alias = network_runner_alias let archive_image = archive_image diff --git a/src/lib/integration_test_abstract_engine/config_util.ml b/src/lib/integration_test_abstract_engine/config_util.ml index 51968ee59c1..f7d4c188c14 100644 --- a/src/lib/integration_test_abstract_engine/config_util.ml +++ b/src/lib/integration_test_abstract_engine/config_util.ml @@ -4,8 +4,8 @@ open Signature_lib open Integration_test_lib module Network_config = Network_config -(* [alias] is instantiated when command line args are parsed *) -let alias = ref None +(* [network_runner_alias] is instantiated when command line args are parsed *) +let network_runner_alias = ref None (* [archive_image] is instantiated when command line args are parsed *) let archive_image : string option ref = ref None @@ -415,9 +415,10 @@ module Config_file = struct let cmd_list = interpolate_args ~args raw_cmd |> String.split ~on:' ' in (List.hd_exn cmd_list, List.tl_exn cmd_list) - let evaluate_alias prog = - match !alias with - | Some (alias, value) when String.(alias = prog) -> + let evaluate_network_runner_alias prog = + match !network_runner_alias with + | Some (network_runner_alias, value) + when String.(network_runner_alias = prog) -> value | _ -> prog @@ -430,7 +431,7 @@ module Config_file = struct let action_args = args_of_action action in let () = assert (validate_args ~args ~action) in let prog, arg_values = prog_and_args ~args action in - let prog = evaluate_alias prog in + let prog = evaluate_network_runner_alias prog in let cmd = String.concat ~sep:" " (prog :: arg_values) in let%map output = Util.run_cmd_or_error_timeout ~suppress_logs ~timeout_seconds dir prog diff --git a/src/lib/integration_test_cloud_engine/kubernetes_network.ml b/src/lib/integration_test_cloud_engine/kubernetes_network.ml index c5de2826d5f..63db3789ea0 100644 --- a/src/lib/integration_test_cloud_engine/kubernetes_network.ml +++ b/src/lib/integration_test_cloud_engine/kubernetes_network.ml @@ -5,7 +5,7 @@ open Integration_test_lib (* exclude from bisect_ppx to avoid type error on GraphQL modules *) [@@@coverage exclude_file] -let alias : (string * string) option ref = ref None +let network_runner_alias : (string * string) option ref = ref None let archive_image : string option ref = ref None diff --git a/src/lib/integration_test_cloud_engine/mina_automation.ml b/src/lib/integration_test_cloud_engine/mina_automation.ml index 084dd2f580c..895ae41bf5f 100644 --- a/src/lib/integration_test_cloud_engine/mina_automation.ml +++ b/src/lib/integration_test_cloud_engine/mina_automation.ml @@ -473,7 +473,7 @@ module Network_config = struct ; region = cluster_region ; zone = Some cluster_zone ; project = Some project_id - ; alias = None + ; network_runner_alias = None } ; Block.Module { Block.Module.local_name = "integration_testnet" diff --git a/src/lib/integration_test_cloud_engine/terraform.ml b/src/lib/integration_test_cloud_engine/terraform.ml index 02eea743983..8d00fdf8807 100644 --- a/src/lib/integration_test_cloud_engine/terraform.ml +++ b/src/lib/integration_test_cloud_engine/terraform.ml @@ -25,19 +25,19 @@ module Block = struct type t = { provider : string ; region : string - ; alias : string option + ; network_runner_alias : string option ; project : string option ; zone : string option } [@@deriving to_yojson] - let to_yojson { provider; region; alias; project; zone } = + let to_yojson { provider; region; network_runner_alias; project; zone } = cons provider () (fun () -> let open Option.Let_syntax in let field k v = (k, `String v) in let fields = [ Some (field "region" region) - ; alias >>| field "alias" + ; network_runner_alias >>| field "network_runner_alias" ; project >>| field "project" ; zone >>| field "zone" ] diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index 5c0c7d5cf5f..76b6366eb46 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -130,8 +130,8 @@ module Engine = struct val id : t -> string - (*** [alias] is instantiated when command line args are parsed *) - val alias : (string * string) option ref + (*** [network_runner_alias] is instantiated when command line args are parsed *) + val network_runner_alias : (string * string) option ref (*** [archive_image] is instantiated when command line args are parsed *) val archive_image : string option ref From 40ca15b37b688b830d6ab3baa024af81940a20a7 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Tue, 12 Sep 2023 21:20:28 +0000 Subject: [PATCH 84/90] (intg test) Add minimina config --- integration_tests/config/minimina.json | 79 +++++++++++++++++++ .../{mock_config.json => config/mock.json} | 0 2 files changed, 79 insertions(+) create mode 100644 integration_tests/config/minimina.json rename integration_tests/{mock_config.json => config/mock.json} (100%) diff --git a/integration_tests/config/minimina.json b/integration_tests/config/minimina.json new file mode 100644 index 00000000000..4a201c1e7f3 --- /dev/null +++ b/integration_tests/config/minimina.json @@ -0,0 +1,79 @@ +{ + "version": 1, + "actions": [ + { + "name": "create_network", + "args": { + "network_id": "string", + "genesis_ledger": "string", + "topology": "string" + }, + "command": "MINIMINA network create --network-id {{network_id}} --genesis-ledger {{genesis_ledger}} --topology {{topology}}" + }, + { + "name": "start_network", + "args": { + "network_id": "string" + }, + "command": "MINIMINA network start --network-id {{network_id}}" + }, + { + "name": "delete_network", + "args": { + "network_id": "string" + }, + "command": "MINIMINA network delete --network-id {{network_id}}" + }, + { + "name": "start_node", + "args": { + "network_id": "string", + "node_id": "string", + "fresh_state": "bool" + }, + "command": "MINIMINA node start {{fresh_state}} --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "stop_node", + "args": { + "network_id": "string", + "node_id": "string" + }, + "command": "MINIMINA node stop --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "dump_archive_data", + "args": { + "network_id": "string", + "node_id": "string" + }, + "command": "MINIMINA node dump-archive-data --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "dump_mina_logs", + "args": { + "network_id": "string", + "node_id": "string" + }, + "command": "MINIMINA node logs --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "dump_precomputed_blocks", + "args": { + "network_id": "string", + "node_id": "string" + }, + "command": "MINIMINA node dump-precomputed-blocks --network-id {{network_id}} --node-id {{node_id}}" + }, + { + "name": "run_replayer", + "args": { + "network_id": "string", + "node_id": "string", + "start_slot_since_genesis": "int" + }, + "command": "MINIMINA node run-replayer --network-id {{network_id}} --node-id {{node_id}} --start-slot-since-genesis {{start_slot_since_genesis}}" + } + ] +} + \ No newline at end of file diff --git a/integration_tests/mock_config.json b/integration_tests/config/mock.json similarity index 100% rename from integration_tests/mock_config.json rename to integration_tests/config/mock.json From 5729e56ac0acf2319e97244a637faa9f7db617a7 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Fri, 15 Sep 2023 03:40:13 +0000 Subject: [PATCH 85/90] (intg test) Handle general network runner alias --- src/app/test_executive/test_executive.ml | 23 ++++++++-------- .../abstract_network.ml | 2 +- .../config_util.ml | 26 +++++++++---------- .../kubernetes_network.ml | 2 +- .../mina_automation.ml | 2 +- .../terraform.ml | 6 ++--- src/lib/integration_test_lib/intf.ml | 4 +-- 7 files changed, 31 insertions(+), 34 deletions(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index f64e0171805..1d93f957923 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -32,7 +32,7 @@ type inputs = ; debug : bool ; config_path : string option ; keypairs_path : string option - ; network_runner_alias : (string * string) option + ; network_runner : string option } let test_name (test : test) @@ -59,7 +59,7 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : in if String.(Inputs.Engine.name = "abstract") - && Option.is_none inputs.network_runner_alias + && Option.is_none inputs.network_runner && (Option.is_none @@ Sys.getenv "MINA_NETWORK_RUNNER") then ( [%log fatal] @@ -80,7 +80,7 @@ let validate_inputs ~logger inputs (test_config : Test_config.t) : exit 1 | Some path -> [%log info] "Using network runner: %s" - (Option.value_exn inputs.network_runner_alias |> snd) ; + (Option.value_exn inputs.network_runner) ; let keypairs_ls = Stdlib.Sys.readdir path in (* check network-keypairs *) if @@ -328,7 +328,7 @@ let main inputs = let error_accumulator_ref = ref None in let network_state_writer_ref = ref None in let cleanup_deferred_ref = ref None in - [%log trace] "preparing up cleanup phase" ; + [%log trace] "preparing cleanup phase" ; let f_dispatch_cleanup = let pause_cleanup_func () = if inputs.debug then @@ -482,7 +482,7 @@ let config_path_arg = & opt (some non_dir_file) None & info [ "config-path"; "config" ] ~env ~docv:"MINA_CI_CONFIG_PATH" ~doc) -let network_runner_alias_arg = +let network_runner_arg = let doc = "Alias for the network runner binary." in let env = Arg.env_var "MINA_NETWORK_RUNNER" ~doc in Arg.( @@ -556,10 +556,9 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = Engine.Network.archive_image := image ; image in - let set_alias alias = - let alias = Option.map alias ~f:(fun a -> ("MOCK_NETWORK", a)) in - Engine.Network.network_runner_alias := alias ; - alias + let set_network_runner network_runner = + Engine.Network.network_runner := network_runner ; + network_runner in let test_inputs_with_cli_inputs_arg = let wrap_cli_inputs cli_inputs = @@ -569,7 +568,7 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = in let inputs_term = let cons_inputs test_inputs test debug archive_image mina_image config_path - keypairs_path network_runner_alias = + keypairs_path network_runner = { test_inputs ; test ; mina_image @@ -577,7 +576,7 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = ; debug ; config_path ; keypairs_path - ; network_runner_alias + ; network_runner } in Term.( @@ -588,7 +587,7 @@ let engine_cmd ((engine_name, (module Engine)) : engine) = $ (const set_mina_image $ mina_image_arg) $ (const set_config $ config_path_arg) $ (const set_keypair $ keypair_dir_path_arg) - $ (const set_alias $ network_runner_alias_arg)) + $ (const set_network_runner $ network_runner_arg)) in (Term.(const start $ inputs_term), info) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index d1841aec210..a89fd565cb2 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -13,7 +13,7 @@ let keypairs_path = keypairs_path let mina_image = mina_image -let network_runner_alias = network_runner_alias +let network_runner = network_runner let archive_image = archive_image diff --git a/src/lib/integration_test_abstract_engine/config_util.ml b/src/lib/integration_test_abstract_engine/config_util.ml index f7d4c188c14..6e518fe7f3f 100644 --- a/src/lib/integration_test_abstract_engine/config_util.ml +++ b/src/lib/integration_test_abstract_engine/config_util.ml @@ -4,8 +4,8 @@ open Signature_lib open Integration_test_lib module Network_config = Network_config -(* [network_runner_alias] is instantiated when command line args are parsed *) -let network_runner_alias = ref None +(* [network_runner] is instantiated when command line args are parsed *) +let network_runner = ref None (* [archive_image] is instantiated when command line args are parsed *) let archive_image : string option ref = ref None @@ -43,7 +43,10 @@ module Node_type = struct match nt with | "Archive_node" | "ArchiveNode" | "Archive" -> Ok Archive_node - | "Block_producer_node" | "BlockProducerNode" | "BlockProducer" -> + | "Block_producer" + | "Block_producer_node" + | "BlockProducerNode" + | "BlockProducer" -> Ok Block_producer_node | "Seed_node" | "SeedNode" | "Seed" -> Ok Seed_node @@ -160,8 +163,8 @@ module Network_deployed = struct | ( node_id , `Assoc [ ("graphql_uri", graphql_uri) - ; ("node_type", `String nt) ; ("private_key", private_key) + ; ("node_type", `String nt) ] ) -> Core.String.Map.set accum ~key:node_id ~data: @@ -415,29 +418,24 @@ module Config_file = struct let cmd_list = interpolate_args ~args raw_cmd |> String.split ~on:' ' in (List.hd_exn cmd_list, List.tl_exn cmd_list) - let evaluate_network_runner_alias prog = - match !network_runner_alias with - | Some (network_runner_alias, value) - when String.(network_runner_alias = prog) -> - value - | _ -> - prog + let evaluate_network_runner prog = Option.value !network_runner ~default:prog - let run_command ?(suppress_logs = false) ?(timeout_seconds = 1) ?(dir = ".") + let run_command ?(suppress_logs = false) ?(timeout_seconds = 420) ?(dir = ".") ~config ~args cmd_name = + let logger = Logger.create () in let config = Yojson.Safe.from_file config in let version = version config in let action = action cmd_name config in let action_args = args_of_action action in let () = assert (validate_args ~args ~action) in let prog, arg_values = prog_and_args ~args action in - let prog = evaluate_network_runner_alias prog in + let prog = evaluate_network_runner prog in let cmd = String.concat ~sep:" " (prog :: arg_values) in + [%log trace] "Running config command: %s" cmd ; let%map output = Util.run_cmd_or_error_timeout ~suppress_logs ~timeout_seconds dir prog arg_values in - let logger = Logger.create () in match output with | Ok output -> if not suppress_logs then diff --git a/src/lib/integration_test_cloud_engine/kubernetes_network.ml b/src/lib/integration_test_cloud_engine/kubernetes_network.ml index 63db3789ea0..ac6cffd937c 100644 --- a/src/lib/integration_test_cloud_engine/kubernetes_network.ml +++ b/src/lib/integration_test_cloud_engine/kubernetes_network.ml @@ -5,7 +5,7 @@ open Integration_test_lib (* exclude from bisect_ppx to avoid type error on GraphQL modules *) [@@@coverage exclude_file] -let network_runner_alias : (string * string) option ref = ref None +let network_runner : string option ref = ref None let archive_image : string option ref = ref None diff --git a/src/lib/integration_test_cloud_engine/mina_automation.ml b/src/lib/integration_test_cloud_engine/mina_automation.ml index 895ae41bf5f..084dd2f580c 100644 --- a/src/lib/integration_test_cloud_engine/mina_automation.ml +++ b/src/lib/integration_test_cloud_engine/mina_automation.ml @@ -473,7 +473,7 @@ module Network_config = struct ; region = cluster_region ; zone = Some cluster_zone ; project = Some project_id - ; network_runner_alias = None + ; alias = None } ; Block.Module { Block.Module.local_name = "integration_testnet" diff --git a/src/lib/integration_test_cloud_engine/terraform.ml b/src/lib/integration_test_cloud_engine/terraform.ml index 8d00fdf8807..02eea743983 100644 --- a/src/lib/integration_test_cloud_engine/terraform.ml +++ b/src/lib/integration_test_cloud_engine/terraform.ml @@ -25,19 +25,19 @@ module Block = struct type t = { provider : string ; region : string - ; network_runner_alias : string option + ; alias : string option ; project : string option ; zone : string option } [@@deriving to_yojson] - let to_yojson { provider; region; network_runner_alias; project; zone } = + let to_yojson { provider; region; alias; project; zone } = cons provider () (fun () -> let open Option.Let_syntax in let field k v = (k, `String v) in let fields = [ Some (field "region" region) - ; network_runner_alias >>| field "network_runner_alias" + ; alias >>| field "alias" ; project >>| field "project" ; zone >>| field "zone" ] diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index 76b6366eb46..b8a38157342 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -130,8 +130,8 @@ module Engine = struct val id : t -> string - (*** [network_runner_alias] is instantiated when command line args are parsed *) - val network_runner_alias : (string * string) option ref + (*** [network_runner] is instantiated when command line args are parsed *) + val network_runner : string option ref (*** [archive_image] is instantiated when command line args are parsed *) val archive_image : string option ref From 6fd877b94a4c81011dd8ac846c0d63a79abd3901 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Fri, 15 Sep 2023 03:41:28 +0000 Subject: [PATCH 86/90] (intg test) Remove fake docker images and git build from mock test --- src/app/test_executive/mock.ml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/app/test_executive/mock.ml b/src/app/test_executive/mock.ml index 9db9d75d21b..415555dafe2 100644 --- a/src/app/test_executive/mock.ml +++ b/src/app/test_executive/mock.ml @@ -21,11 +21,10 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let i_str = Int.to_string i in test_account ("sender-account-" ^ i_str) "10000" ) ; block_producers = - [ bp "receiver" ~docker_image:"receiver-docker-image" () + [ bp "receiver" () ; bp "empty_node-1" ~account_name:"empty-bp-key" () ; bp "empty_node-2" ~account_name:"empty-bp-key" () - ; bp "observer" ~account_name:"empty-bp-key" - ~git_build:(Commit "abcdef0123") () + ; bp "observer" ~account_name:"empty-bp-key" () ] ; snark_coordinator = snark "snark-node" 4 ; txpool_max_size = 10_000_000 @@ -36,8 +35,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ; transaction_capacity = Some Runtime_config.Proof_keys.Transaction_capacity.small } - ; seed_nodes = - [ seed "seed-0" ~docker_image:"seed-docker-image" (); seed "seed-1" () ] + ; seed_nodes = [ seed "seed-0" (); seed "seed-1" () ] ; archive_nodes = [ archive "archive-node" () ] } From 8f45f43cea56c748e25478ecc14a6c69bda492f3 Mon Sep 17 00:00:00 2001 From: Quantifier Date: Fri, 15 Sep 2023 17:18:59 +0000 Subject: [PATCH 87/90] (intg test) Improve logging --- .../config_util.ml | 6 ++---- .../mina_automation.ml | 14 +++++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/lib/integration_test_abstract_engine/config_util.ml b/src/lib/integration_test_abstract_engine/config_util.ml index 6e518fe7f3f..0bfde349ab6 100644 --- a/src/lib/integration_test_abstract_engine/config_util.ml +++ b/src/lib/integration_test_abstract_engine/config_util.ml @@ -431,16 +431,14 @@ module Config_file = struct let prog, arg_values = prog_and_args ~args action in let prog = evaluate_network_runner prog in let cmd = String.concat ~sep:" " (prog :: arg_values) in - [%log trace] "Running config command: %s" cmd ; + if not suppress_logs then [%log trace] "Running config command: %s" cmd ; let%map output = Util.run_cmd_or_error_timeout ~suppress_logs ~timeout_seconds dir prog arg_values in match output with | Ok output -> - if not suppress_logs then - [%log spam] "Successful command execution\n Command: %s\n Output: %s" - cmd output ; + if not suppress_logs then [%log trace] "Output: %s" output ; Ok output | Error err -> if not suppress_logs then [%log error] "Failed to run command: %s" cmd ; diff --git a/src/lib/integration_test_abstract_engine/mina_automation.ml b/src/lib/integration_test_abstract_engine/mina_automation.ml index 16341adeaff..771f82dcba8 100644 --- a/src/lib/integration_test_abstract_engine/mina_automation.ml +++ b/src/lib/integration_test_abstract_engine/mina_automation.ml @@ -193,13 +193,17 @@ module Network_manager = struct |> Fn.compose (String.concat ~sep:", ") (List.map ~f:Abstract_network.Node.id) in + let report_nodes nodes kind = + [%log info] "%s: %s" kind + ( if not @@ String.Map.is_empty nodes then nodes_to_string nodes + else sprintf "No %s" kind ) + in [%log info] "Network deployed" ; [%log info] "network id: %s" t.network_id ; - [%log info] "snark coordinators: %s" - (nodes_to_string network.snark_coordinators) ; - [%log info] "snark workers: %s" (nodes_to_string network.snark_workers) ; - [%log info] "block producers: %s" (nodes_to_string network.block_producers) ; - [%log info] "archive nodes: %s" (nodes_to_string network.archive_nodes) ; + report_nodes network.snark_coordinators "snark coordinator" ; + report_nodes network.snark_workers "snark workers" ; + report_nodes network.block_producers "block producers" ; + report_nodes network.archive_nodes "archive nodes" ; Malleable_error.return network let destroy t = From f387b0894767c2de397ac5f7facf465fd96863ee Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Tue, 19 Sep 2023 15:15:31 +0100 Subject: [PATCH 88/90] Fix payments test --- src/app/test_executive/payments_test.ml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/app/test_executive/payments_test.ml b/src/app/test_executive/payments_test.ml index a8a38645db9..1ce1cd43c75 100644 --- a/src/app/test_executive/payments_test.ml +++ b/src/app/test_executive/payments_test.ml @@ -45,8 +45,9 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct ] ; block_producers = [ bp "untimed-node-a" (); bp "untimed-node-b" (); bp "timed-node-c" () ] - ; snark_coordinator = snark "snark-node" 4 - ; snark_worker_fee = "0.0001" + ; snark_coordinator = snark "snark-node" ~account_name:"snark-node-key1" 4 + ; snark_worker_fee = "0.0002" + ; archive_nodes = [ archive "archive-node" () ] ; proof_config = { proof_config_default with work_delay = Some 1 From afc6fac265b6078ed81ecc9994b82d1a57ff6761 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Fri, 22 Sep 2023 08:50:15 +0100 Subject: [PATCH 89/90] Remove duplicate uri entry in dune profile --- src/app/test_executive/dune | 124 ++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/src/app/test_executive/dune b/src/app/test_executive/dune index 37e12a8ce17..a18189eb2b9 100644 --- a/src/app/test_executive/dune +++ b/src/app/test_executive/dune @@ -1,65 +1,65 @@ (executable (name test_executive) (libraries - ;; opam libraries - async_kernel - async - core - uri - yojson - core_kernel - cmdliner - base.base_internalhash_types - base.caml - async_unix - unsigned_extended - stdio - uri - ;; local libraries - mina_wire_types - with_hash - data_hash_lib - kimchi_backend - kimchi_backend.pasta - kimchi_backend.pasta.basic - pickles - pickles_types - random_oracle_input - genesis_constants - bash_colors - integration_test_lib - signature_lib - mina_signature_kind - mina_base - mina_stdlib - mina_transaction - file_system - currency - mina_runtime_config - secrets - integration_test_abstract_engine - integration_test_cloud_engine - mina_generators - logger - random_oracle - mina_numbers - transaction_snark - snark_params - pickles.backend - pipe_lib - mina_base.import - key_gen - integers - user_command_input - participating_state - graph_algorithms - visualization - sgn - zkapp_command_builder - network_pool - zkapps_examples - cache_dir - snarky.backendless - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_jane ppx_deriving_yojson ppx_mina ppx_version))) + ;; opam libraries + async_kernel + async + core + uri + yojson + core_kernel + cmdliner + base.base_internalhash_types + base.caml + async_unix + unsigned_extended + stdio + ;; local libraries + mina_wire_types + with_hash + data_hash_lib + kimchi_backend + kimchi_backend.pasta + kimchi_backend.pasta.basic + pickles + pickles_types + random_oracle_input + genesis_constants + bash_colors + integration_test_lib + signature_lib + mina_signature_kind + mina_base + mina_stdlib + mina_transaction + file_system + currency + mina_runtime_config + secrets + integration_test_abstract_engine + integration_test_cloud_engine + mina_generators + logger + random_oracle + mina_numbers + transaction_snark + snark_params + pickles.backend + pipe_lib + mina_base.import + key_gen + integers + user_command_input + participating_state + graph_algorithms + visualization + sgn + zkapp_command_builder + network_pool + zkapps_examples + cache_dir + snarky.backendless) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_mina ppx_jane ppx_deriving_yojson ppx_mina ppx_version))) From a40c5eed0898cf823f326bd7a63570cd34247522 Mon Sep 17 00:00:00 2001 From: ejMina226 <118474890+ejMina226@users.noreply.github.com> Date: Fri, 22 Sep 2023 09:51:45 +0100 Subject: [PATCH 90/90] Resolving Rebase Conflicts --- src/app/test_executive/test_executive.ml | 1 - .../abstract_network.ml | 975 +----------------- .../graphql_polling_log_engine.ml | 10 +- .../network_config.ml | 1 + .../mina_automation.ml | 4 +- src/lib/integration_test_lib/intf.ml | 29 - src/lib/integration_test_lib/test_config.ml | 3 +- 7 files changed, 19 insertions(+), 1004 deletions(-) diff --git a/src/app/test_executive/test_executive.ml b/src/app/test_executive/test_executive.ml index 1d93f957923..40091a8b974 100644 --- a/src/app/test_executive/test_executive.ml +++ b/src/app/test_executive/test_executive.ml @@ -121,7 +121,6 @@ let tests : test list = [ (module Block_production_priority.Make : Intf.Test.Functor_intf) ; (module Block_reward_test.Make : Intf.Test.Functor_intf) ; (module Chain_reliability_test.Make : Intf.Test.Functor_intf) - ; (module Delegation_test.Make : Intf.Test.Functor_intf) ; (module Gossip_consistency.Make : Intf.Test.Functor_intf) ; (module Medium_bootstrap.Make : Intf.Test.Functor_intf) ; (module Mock.Make : Intf.Test.Functor_intf) diff --git a/src/lib/integration_test_abstract_engine/abstract_network.ml b/src/lib/integration_test_abstract_engine/abstract_network.ml index a89fd565cb2..fad8c86839e 100644 --- a/src/lib/integration_test_abstract_engine/abstract_network.ml +++ b/src/lib/integration_test_abstract_engine/abstract_network.ml @@ -1,7 +1,6 @@ open Core_kernel open Async open Integration_test_lib -open Mina_transaction open Config_util (* exclude from bisect_ppx to avoid type error on GraphQL modules *) @@ -31,6 +30,8 @@ module Node = struct let network_keypair { network_keypair; _ } = network_keypair + let get_ingress_uri node = Uri.of_string node.graphql_uri + let start ~fresh_state t : unit Malleable_error.t = try let args = @@ -105,972 +106,6 @@ module Node = struct filter ~f:is_snark_worker t |> map ~f:(f @@ network_id t) end - module Scalars = Graphql_lib.Scalars - - module Graphql = struct - let ingress_uri node = Uri.of_string node.graphql_uri - - module Client = Graphql_lib.Client.Make (struct - let preprocess_variables_string = Fn.id - - let headers = String.Map.empty - end) - - (* graphql_ppx uses Stdlib symbols instead of Base *) - open Stdlib - module Encoders = Mina_graphql.Types.Input - - module Unlock_account = - [%graphql - ({| - mutation ($password: String!, $public_key: PublicKey!) @encoders(module: "Encoders"){ - unlockAccount(input: {password: $password, publicKey: $public_key }) { - account { - public_key: publicKey - } - } - } - |} - [@encoders Encoders] )] - - module Send_test_payments = - [%graphql - {| - mutation ($senders: [PrivateKey!]!, - $receiver: PublicKey!, - $amount: UInt64!, - $fee: UInt64!, - $repeat_count: UInt32!, - $repeat_delay_ms: UInt32!) @encoders(module: "Encoders"){ - sendTestPayments( - senders: $senders, receiver: $receiver, amount: $amount, fee: $fee, - repeat_count: $repeat_count, - repeat_delay_ms: $repeat_delay_ms) - } - |}] - - module Send_payment = - [%graphql - {| - mutation ($input: SendPaymentInput!)@encoders(module: "Encoders"){ - sendPayment(input: $input){ - payment { - id - nonce - hash - } - } - } - |}] - - module Send_payment_with_raw_sig = - [%graphql - {| - mutation ( - $input:SendPaymentInput!, - $rawSignature: String! - )@encoders(module: "Encoders") - { - sendPayment( - input:$input, - signature: {rawSignature: $rawSignature} - ) - { - payment { - id - nonce - hash - } - } - } - |}] - - module Send_delegation = - [%graphql - {| - mutation ($input: SendDelegationInput!) @encoders(module: "Encoders"){ - sendDelegation(input:$input){ - delegation { - id - nonce - hash - } - } - } - |}] - - module Set_snark_worker = - [%graphql - {| - mutation ($input: SetSnarkWorkerInput! ) @encoders(module: "Encoders"){ - setSnarkWorker(input:$input){ - lastSnarkWorker - } - } - |}] - - module Get_account_data = - [%graphql - {| - query ($public_key: PublicKey!) @encoders(module: "Encoders"){ - account(publicKey: $public_key) { - nonce - balance { - total @ppxCustom(module: "Scalars.Balance") - } - } - } - |}] - - module Send_test_zkapp = Generated_graphql_queries.Send_test_zkapp - module Pooled_zkapp_commands = - Generated_graphql_queries.Pooled_zkapp_commands - - module Query_peer_id = - [%graphql - {| - query { - daemonStatus { - addrsAndPorts { - peer { - peerId - } - } - peers { peerId } - - } - } - |}] - - module Best_chain = - [%graphql - {| - query ($max_length: Int) @encoders(module: "Encoders"){ - bestChain (maxLength: $max_length) { - stateHash @ppxCustom(module: "Graphql_lib.Scalars.String_json") - commandTransactionCount - creatorAccount { - publicKey @ppxCustom(module: "Graphql_lib.Scalars.JSON") - } - } - } - |}] - - module Query_metrics = - [%graphql - {| - query { - daemonStatus { - metrics { - blockProductionDelay - transactionPoolDiffReceived - transactionPoolDiffBroadcasted - transactionsAddedToPool - transactionPoolSize - } - } - } - |}] - - module StartFilteredLog = - [%graphql - {| - mutation ($filter: [String!]!) @encoders(module: "Encoders"){ - startFilteredLog(filter: $filter) - } - |}] - - module GetFilteredLogEntries = - [%graphql - {| - query ($offset: Int!) @encoders(module: "Encoders"){ - getFilteredLogEntries(offset: $offset) { - logMessages, - isCapturing, - } - - } - |}] - - module Account = - [%graphql - {| - query ($public_key: PublicKey!, $token: UInt64) { - account (publicKey : $public_key, token : $token) { - balance { liquid - locked - total - } - delegate - nonce - permissions { editActionState - editState - incrementNonce - receive - send - access - setDelegate - setPermissions - setZkappUri - setTokenSymbol - setVerificationKey - setVotingFor - setTiming - } - actionState - zkappState - zkappUri - timing { cliffTime @ppxCustom(module: "Graphql_lib.Scalars.JSON") - cliffAmount - vestingPeriod @ppxCustom(module: "Graphql_lib.Scalars.JSON") - vestingIncrement - initialMinimumBalance - } - tokenId - tokenSymbol - verificationKey { verificationKey - hash - } - votingFor - } - } - |}] - end - - (* this function will repeatedly attempt to connect to graphql port times before giving up *) - let exec_graphql_request ?(num_tries = 10) ?(retry_delay_sec = 30.0) - ?(initial_delay_sec = 0.) ~logger ~node ~query_name query_obj = - let open Deferred.Let_syntax in - let uri = Graphql.ingress_uri node in - let metadata = - [ ("query", `String query_name) - ; ("uri", `String (Uri.to_string uri)) - ; ("init_delay", `Float initial_delay_sec) - ] - in - [%log info] - "Attempting to send GraphQL request \"$query\" to \"$uri\" after \ - $init_delay sec" - ~metadata ; - let rec retry n = - if n <= 0 then ( - [%log error] - "GraphQL request \"$query\" to \"$uri\" failed too many times" - ~metadata ; - Deferred.Or_error.errorf - "GraphQL \"%s\" to \"%s\" request failed too many times" query_name - (Uri.to_string uri) ) - else - match%bind Graphql.Client.query query_obj uri with - | Ok result -> - [%log info] "GraphQL request \"$query\" to \"$uri\" succeeded" - ~metadata ; - Deferred.Or_error.return result - | Error (`Failed_request err_string) -> - [%log warn] - "GraphQL request \"$query\" to \"$uri\" failed: \"$error\" \ - ($num_tries attempts left)" - ~metadata: - ( metadata - @ [ ("error", `String err_string); ("num_tries", `Int (n - 1)) ] - ) ; - let%bind () = after (Time.Span.of_sec retry_delay_sec) in - retry (n - 1) - | Error (`Graphql_error err_string) -> - [%log error] - "GraphQL request \"$query\" to \"$uri\" returned an error: \ - \"$error\" (this is a graphql error so not retrying)" - ~metadata:(("error", `String err_string) :: metadata) ; - Deferred.Or_error.error_string err_string - in - let%bind () = after (Time.Span.of_sec initial_delay_sec) in - retry num_tries - - let get_peer_id ~logger t = - let open Deferred.Or_error.Let_syntax in - [%log info] "Getting node's peer_id, and the peer_ids of node's peers" - ~metadata:(logger_metadata t) ; - let query_obj = Graphql.Query_peer_id.(make @@ makeVariables ()) in - let%bind query_result_obj = - exec_graphql_request ~logger ~node:t ~query_name:"query_peer_id" query_obj - in - [%log info] "get_peer_id, finished exec_graphql_request" ; - let self_id_obj = query_result_obj.daemonStatus.addrsAndPorts.peer in - let%bind self_id = - match self_id_obj with - | None -> - Deferred.Or_error.error_string "Peer not found" - | Some peer -> - return peer.peerId - in - let peers = query_result_obj.daemonStatus.peers |> Array.to_list in - let peer_ids = List.map peers ~f:(fun peer -> peer.peerId) in - [%log info] "get_peer_id, result of graphql query (self_id,[peers]) (%s,%s)" - self_id - (String.concat ~sep:" " peer_ids) ; - return (self_id, peer_ids) - - let must_get_peer_id ~logger t = - get_peer_id ~logger t |> Deferred.bind ~f:Malleable_error.or_hard_error - - let get_best_chain ?max_length ~logger t = - let open Deferred.Or_error.Let_syntax in - let query = Graphql.Best_chain.(make @@ makeVariables ?max_length ()) in - let%bind result = - exec_graphql_request ~logger ~node:t ~query_name:"best_chain" query - in - match result.bestChain with - | None | Some [||] -> - Deferred.Or_error.error_string "failed to get best chains" - | Some chain -> - return - @@ List.map - ~f:(fun block -> - Intf. - { state_hash = block.stateHash - ; command_transaction_count = block.commandTransactionCount - ; creator_pk = - ( match block.creatorAccount.publicKey with - | `String pk -> - pk - | _ -> - "unknown" ) - } ) - (Array.to_list chain) - - let must_get_best_chain ?max_length ~logger t = - get_best_chain ?max_length ~logger t - |> Deferred.bind ~f:Malleable_error.or_hard_error - - let get_account ~logger t ~account_id = - let pk = Mina_base.Account_id.public_key account_id in - let token = Mina_base.Account_id.token_id account_id in - [%log info] "Getting account" - ~metadata: - ( ("pub_key", Signature_lib.Public_key.Compressed.to_yojson pk) - :: logger_metadata t ) ; - let get_account_obj = - Graphql.Account.( - make - @@ makeVariables - ~public_key:(Graphql_lib.Encoders.public_key pk) - ~token:(Graphql_lib.Encoders.token token) - ()) - in - exec_graphql_request ~logger ~node:t ~query_name:"get_account_graphql" - get_account_obj - - type account_data = - { nonce : Mina_numbers.Account_nonce.t - ; total_balance : Currency.Balance.t - ; liquid_balance_opt : Currency.Balance.t option - ; locked_balance_opt : Currency.Balance.t option - } - - let get_account_data ~logger t ~account_id = - let open Deferred.Or_error.Let_syntax in - let public_key = Mina_base.Account_id.public_key account_id in - let token = Mina_base.Account_id.token_id account_id in - [%log info] "Getting account data, which is its balances and nonce" - ~metadata: - ( ("pub_key", Signature_lib.Public_key.Compressed.to_yojson public_key) - :: logger_metadata t ) ; - let%bind account_obj = get_account ~logger t ~account_id in - match account_obj.account with - | None -> - Deferred.Or_error.errorf - !"Account with Account id %{sexp:Mina_base.Account_id.t}, public_key \ - %s, and token %s not found" - account_id - (Signature_lib.Public_key.Compressed.to_string public_key) - (Mina_base.Token_id.to_string token) - | Some acc -> - return - { nonce = - Option.value_exn - ~message: - "the nonce from get_balance is None, which should be \ - impossible" - acc.nonce - ; total_balance = acc.balance.total - ; liquid_balance_opt = acc.balance.liquid - ; locked_balance_opt = acc.balance.locked - } - - let must_get_account_data ~logger t ~account_id = - get_account_data ~logger t ~account_id - |> Deferred.bind ~f:Malleable_error.or_hard_error - - let permissions_of_account_permissions account_permissions : - Mina_base.Permissions.t = - (* the polymorphic variants come from Partial_accounts.auth_required in Mina_graphql *) - let to_auth_required = function - | `Either -> - Mina_base.Permissions.Auth_required.Either - | `Impossible -> - Impossible - | `None -> - None - | `Proof -> - Proof - | `Signature -> - Signature - in - let open Graphql.Account in - { edit_action_state = to_auth_required account_permissions.editActionState - ; edit_state = to_auth_required account_permissions.editState - ; increment_nonce = to_auth_required account_permissions.incrementNonce - ; receive = to_auth_required account_permissions.receive - ; send = to_auth_required account_permissions.send - ; access = to_auth_required account_permissions.access - ; set_delegate = to_auth_required account_permissions.setDelegate - ; set_permissions = to_auth_required account_permissions.setPermissions - ; set_zkapp_uri = to_auth_required account_permissions.setZkappUri - ; set_token_symbol = to_auth_required account_permissions.setTokenSymbol - ; set_verification_key = - to_auth_required account_permissions.setVerificationKey - ; set_voting_for = to_auth_required account_permissions.setVotingFor - ; set_timing = to_auth_required account_permissions.setTiming - } - - let graphql_uri node = Graphql.ingress_uri node |> Uri.to_string - - let get_account_permissions ~logger t ~account_id = - let open Deferred.Or_error in - let open Let_syntax in - let%bind account_obj = get_account ~logger t ~account_id in - match account_obj.account with - | Some account -> ( - match account.permissions with - | Some ledger_permissions -> - return @@ permissions_of_account_permissions ledger_permissions - | None -> - fail - (Error.of_string "Could not get permissions from ledger account") - ) - | None -> - fail (Error.of_string "Could not get account from ledger") - - (* return an Account_update.Update.t with all fields `Set` to the - value in the account, or `Keep` if value unavailable, - as if this update had been applied to the account - *) - let get_account_update ~logger t ~account_id = - let open Deferred.Or_error in - let open Let_syntax in - let%bind account_obj = get_account ~logger t ~account_id in - match account_obj.account with - | Some account -> - let open Mina_base.Zkapp_basic.Set_or_keep in - let%bind app_state = - match account.zkappState with - | Some strs -> - let fields = - Array.to_list strs |> Base.List.map ~f:(fun s -> Set s) - in - return (Mina_base.Zkapp_state.V.of_list_exn fields) - | None -> - fail - (Error.of_string - (sprintf - "Expected zkApp account with an app state for public key \ - %s" - (Signature_lib.Public_key.Compressed.to_base58_check - (Mina_base.Account_id.public_key account_id) ) ) ) - in - let%bind delegate = - match account.delegate with - | Some s -> - return (Set s) - | None -> - fail (Error.of_string "Expected delegate in account") - in - let%bind verification_key = - match account.verificationKey with - | Some vk_obj -> - let data = vk_obj.verificationKey in - let hash = vk_obj.hash in - return (Set ({ data; hash } : _ With_hash.t)) - | None -> - fail - (Error.of_string - (sprintf - "Expected zkApp account with a verification key for \ - public_key %s" - (Signature_lib.Public_key.Compressed.to_base58_check - (Mina_base.Account_id.public_key account_id) ) ) ) - in - let%bind permissions = - match account.permissions with - | Some perms -> - return @@ Set (permissions_of_account_permissions perms) - | None -> - fail (Error.of_string "Expected permissions in account") - in - let%bind zkapp_uri = - match account.zkappUri with - | Some s -> - return @@ Set s - | None -> - fail (Error.of_string "Expected zkApp URI in account") - in - let%bind token_symbol = - match account.tokenSymbol with - | Some s -> - return @@ Set s - | None -> - fail (Error.of_string "Expected token symbol in account") - in - let%bind timing = - let timing = account.timing in - let cliff_amount = timing.cliffAmount in - let cliff_time = timing.cliffTime in - let vesting_period = timing.vestingPeriod in - let vesting_increment = timing.vestingIncrement in - let initial_minimum_balance = timing.initialMinimumBalance in - match - ( cliff_amount - , cliff_time - , vesting_period - , vesting_increment - , initial_minimum_balance ) - with - | None, None, None, None, None -> - return @@ Keep - | Some amt, Some tm, Some period, Some incr, Some bal -> - let cliff_amount = amt in - let%bind cliff_time = - match tm with - | `String s -> - return @@ Mina_numbers.Global_slot_since_genesis.of_string s - | _ -> - fail - (Error.of_string - "Expected string for cliff time in account timing" ) - in - let%bind vesting_period = - match period with - | `String s -> - return @@ Mina_numbers.Global_slot_span.of_string s - | _ -> - fail - (Error.of_string - "Expected string for vesting period in account timing" ) - in - let vesting_increment = incr in - let initial_minimum_balance = bal in - return - (Set - ( { initial_minimum_balance - ; cliff_amount - ; cliff_time - ; vesting_period - ; vesting_increment - } - : Mina_base.Account_update.Update.Timing_info.t ) ) - | _ -> - fail (Error.of_string "Some pieces of account timing are missing") - in - let%bind voting_for = - match account.votingFor with - | Some s -> - return @@ Set s - | None -> - fail (Error.of_string "Expected voting-for state hash in account") - in - return - ( { app_state - ; delegate - ; verification_key - ; permissions - ; zkapp_uri - ; token_symbol - ; timing - ; voting_for - } - : Mina_base.Account_update.Update.t ) - | None -> - fail (Error.of_string "Could not get account from ledger") - - type signed_command_result = - { id : string - ; hash : Transaction_hash.t - ; nonce : Mina_numbers.Account_nonce.t - } - - let transaction_id_to_string id = - Yojson.Basic.to_string (Graphql_lib.Scalars.TransactionId.serialize id) - - (* if we expect failure, might want retry_on_graphql_error to be false *) - let send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee = - [%log info] "Sending a payment" ~metadata:(logger_metadata t) ; - let open Deferred.Or_error.Let_syntax in - let sender_pk_str = - Signature_lib.Public_key.Compressed.to_string sender_pub_key - in - [%log info] "send_payment: unlocking account" - ~metadata:[ ("sender_pk", `String sender_pk_str) ] ; - let unlock_sender_account_graphql () = - let unlock_account_obj = - Graphql.Unlock_account.( - make - @@ makeVariables ~password:t.password ~public_key:sender_pub_key ()) - in - exec_graphql_request ~logger ~node:t ~initial_delay_sec:0. - ~query_name:"unlock_sender_account_graphql" unlock_account_obj - in - let%bind _unlock_acct_obj = unlock_sender_account_graphql () in - let send_payment_graphql () = - let input = - Mina_graphql.Types.Input.SendPaymentInput.make_input - ~from:sender_pub_key ~to_:receiver_pub_key ~amount ~fee () - in - let send_payment_obj = - Graphql.Send_payment.(make @@ makeVariables ~input ()) - in - exec_graphql_request ~logger ~node:t ~query_name:"send_payment_graphql" - send_payment_obj - in - let%map sent_payment_obj = send_payment_graphql () in - let return_obj = sent_payment_obj.sendPayment.payment in - let res = - { id = transaction_id_to_string return_obj.id - ; hash = return_obj.hash - ; nonce = Mina_numbers.Account_nonce.of_int return_obj.nonce - } - in - [%log info] "Sent payment" - ~metadata: - [ ("user_command_id", `String res.id) - ; ("hash", `String (Transaction_hash.to_base58_check res.hash)) - ; ("nonce", `Int (Mina_numbers.Account_nonce.to_int res.nonce)) - ] ; - res - - let must_send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee - = - send_payment ~logger t ~sender_pub_key ~receiver_pub_key ~amount ~fee - |> Deferred.bind ~f:Malleable_error.or_hard_error - - let send_zkapp_batch ~logger (t : t) - ~(zkapp_commands : Mina_base.Zkapp_command.t list) = - [%log info] "Sending zkapp transactions" - ~metadata: - [ ("network_id", `String t.network_id); ("node_id", `String t.node_id) ] ; - let open Deferred.Or_error.Let_syntax in - let zkapp_commands_json = - List.map zkapp_commands ~f:(fun zkapp_command -> - Mina_base.Zkapp_command.to_json zkapp_command |> Yojson.Safe.to_basic ) - |> Array.of_list - in - let send_zkapp_graphql () = - let send_zkapp_obj = - Graphql.Send_test_zkapp.( - make @@ makeVariables ~zkapp_commands:zkapp_commands_json ()) - in - exec_graphql_request ~logger ~node:t ~query_name:"send_zkapp_graphql" - send_zkapp_obj - in - let%bind sent_zkapp_obj = send_zkapp_graphql () in - let%bind zkapp_ids = - Deferred.Array.fold ~init:(Ok []) sent_zkapp_obj.internalSendZkapp - ~f:(fun acc (zkapp_obj : Graphql.Send_test_zkapp.t_internalSendZkapp) -> - let%bind res = - match zkapp_obj.zkapp.failureReason with - | None -> - let zkapp_id = transaction_id_to_string zkapp_obj.zkapp.id in - [%log info] "Sent zkapp transaction" - ~metadata:[ ("zkapp_id", `String zkapp_id) ] ; - return zkapp_id - | Some s -> - Deferred.Or_error.errorf "Zkapp failed, reason: %s" - ( Array.fold ~init:[] s ~f:(fun acc f -> - match f with - | None -> - acc - | Some f -> - let t = - ( Option.value_exn f.index - , f.failures |> Array.to_list |> List.rev ) - in - t :: acc ) - |> Mina_base.Transaction_status.Failure.Collection.Display - .to_yojson |> Yojson.Safe.to_string ) - in - let%map acc = Deferred.return acc in - res :: acc ) - in - return (List.rev zkapp_ids) - - let get_pooled_zkapp_commands ~logger (t : t) - ~(pk : Signature_lib.Public_key.Compressed.t) = - [%log info] "Retrieving zkapp_commands from transaction pool" - ~metadata: - [ ("network_id", `String t.network_id) - ; ("node_id", `String t.node_id) - ; ("pub_key", Signature_lib.Public_key.Compressed.to_yojson pk) - ] ; - let open Deferred.Or_error.Let_syntax in - let get_pooled_zkapp_commands_graphql () = - let get_pooled_zkapp_commands = - Graphql.Pooled_zkapp_commands.( - make - @@ makeVariables ~public_key:(Graphql_lib.Encoders.public_key pk) ()) - in - exec_graphql_request ~logger ~node:t - ~query_name:"get_pooled_zkapp_commands" get_pooled_zkapp_commands - in - let%bind zkapp_pool_obj = get_pooled_zkapp_commands_graphql () in - let transaction_ids = - Array.map zkapp_pool_obj.pooledZkappCommands ~f:(fun zkapp_command -> - zkapp_command.id |> Transaction_id.to_base64 ) - |> Array.to_list - in - [%log info] "Retrieved zkapp_commands from transaction pool" - ~metadata: - [ ("network_id", `String t.network_id) - ; ("node_id", `String t.node_id) - ; ( "transaction ids" - , `List (List.map ~f:(fun t -> `String t) transaction_ids) ) - ] ; - return transaction_ids - - let send_delegation ~logger t ~sender_pub_key ~receiver_pub_key ~fee = - [%log info] "Sending stake delegation" ~metadata:(logger_metadata t) ; - let open Deferred.Or_error.Let_syntax in - let sender_pk_str = - Signature_lib.Public_key.Compressed.to_string sender_pub_key - in - [%log info] "send_delegation: unlocking account" - ~metadata:[ ("sender_pk", `String sender_pk_str) ] ; - let unlock_sender_account_graphql () = - let unlock_account_obj = - Graphql.Unlock_account.( - make - @@ makeVariables ~password:t.password ~public_key:sender_pub_key ()) - in - exec_graphql_request ~logger ~node:t - ~query_name:"unlock_sender_account_graphql" unlock_account_obj - in - let%bind _ = unlock_sender_account_graphql () in - let send_delegation_graphql () = - let input = - Mina_graphql.Types.Input.SendDelegationInput.make_input - ~from:sender_pub_key ~to_:receiver_pub_key ~fee () - in - let send_delegation_obj = - Graphql.Send_delegation.(make @@ makeVariables ~input ()) - in - exec_graphql_request ~logger ~node:t ~query_name:"send_delegation_graphql" - send_delegation_obj - in - let%map result_obj = send_delegation_graphql () in - let return_obj = result_obj.sendDelegation.delegation in - let res = - { id = transaction_id_to_string return_obj.id - ; hash = return_obj.hash - ; nonce = Mina_numbers.Account_nonce.of_int return_obj.nonce - } - in - [%log info] "stake delegation sent" - ~metadata: - [ ("user_command_id", `String res.id) - ; ("hash", `String (Transaction_hash.to_base58_check res.hash)) - ; ("nonce", `Int (Mina_numbers.Account_nonce.to_int res.nonce)) - ] ; - res - - let must_send_delegation ~logger t ~sender_pub_key ~receiver_pub_key ~fee = - send_delegation ~logger t ~sender_pub_key ~receiver_pub_key ~fee - |> Deferred.bind ~f:Malleable_error.or_hard_error - - let send_payment_with_raw_sig ~logger t ~sender_pub_key ~receiver_pub_key - ~amount ~fee ~nonce ~memo - ~(valid_until : Mina_numbers.Global_slot_since_genesis.t) ~raw_signature = - [%log info] "Sending a payment with raw signature" - ~metadata:(logger_metadata t) ; - let open Deferred.Or_error.Let_syntax in - let send_payment_graphql () = - let open Graphql.Send_payment_with_raw_sig in - let input = - Mina_graphql.Types.Input.SendPaymentInput.make_input - ~from:sender_pub_key ~to_:receiver_pub_key ~amount ~fee ~memo ~nonce - ~valid_until: - (Mina_numbers.Global_slot_since_genesis.to_uint32 valid_until) - () - in - let variables = makeVariables ~input ~rawSignature:raw_signature () in - let send_payment_obj = make variables in - let variables_json_basic = - variablesToJson (serializeVariables variables) - in - (* An awkward conversion from Yojson.Basic to Yojson.Safe *) - let variables_json = - Yojson.Basic.to_string variables_json_basic |> Yojson.Safe.from_string - in - [%log info] "send_payment_obj with $variables " - ~metadata:[ ("variables", variables_json) ] ; - exec_graphql_request ~logger ~node:t - ~query_name:"Send_payment_with_raw_sig_graphql" send_payment_obj - in - let%map sent_payment_obj = send_payment_graphql () in - let return_obj = sent_payment_obj.sendPayment.payment in - let res = - { id = transaction_id_to_string return_obj.id - ; hash = return_obj.hash - ; nonce = Mina_numbers.Account_nonce.of_int return_obj.nonce - } - in - [%log info] "Sent payment" - ~metadata: - [ ("user_command_id", `String res.id) - ; ("hash", `String (Transaction_hash.to_base58_check res.hash)) - ; ("nonce", `Int (Mina_numbers.Account_nonce.to_int res.nonce)) - ] ; - res - - let must_send_payment_with_raw_sig ~logger t ~sender_pub_key ~receiver_pub_key - ~amount ~fee ~nonce ~memo ~valid_until ~raw_signature = - send_payment_with_raw_sig ~logger t ~sender_pub_key ~receiver_pub_key - ~amount ~fee ~nonce ~memo ~valid_until ~raw_signature - |> Deferred.bind ~f:Malleable_error.or_hard_error - - let send_test_payments ~repeat_count ~repeat_delay_ms ~logger t ~senders - ~receiver_pub_key ~amount ~fee = - [%log info] "Sending a series of test payments" - ~metadata:(logger_metadata t) ; - let open Deferred.Or_error.Let_syntax in - let send_payment_graphql () = - let send_payment_obj = - Graphql.Send_test_payments.( - make - @@ makeVariables ~senders ~receiver:receiver_pub_key - ~amount:(Currency.Amount.to_uint64 amount) - ~fee:(Currency.Fee.to_uint64 fee) - ~repeat_count ~repeat_delay_ms ()) - in - exec_graphql_request ~logger ~node:t ~query_name:"send_payment_graphql" - send_payment_obj - in - let%map _ = send_payment_graphql () in - [%log info] "Sent test payments" - - let must_send_test_payments ~repeat_count ~repeat_delay_ms ~logger t ~senders - ~receiver_pub_key ~amount ~fee = - send_test_payments ~repeat_count ~repeat_delay_ms ~logger t ~senders - ~receiver_pub_key ~amount ~fee - |> Deferred.bind ~f:Malleable_error.or_hard_error - - let set_snark_worker ~logger t ~new_snark_pub_key = - [%log info] "Changing snark worker key" ~metadata:(logger_metadata t) ; - let open Deferred.Or_error.Let_syntax in - let set_snark_worker_graphql () = - let input = Some new_snark_pub_key in - let set_snark_worker_obj = - Graphql.Set_snark_worker.(make @@ makeVariables ~input ()) - in - exec_graphql_request ~logger ~node:t - ~query_name:"set_snark_worker_graphql" set_snark_worker_obj - in - let%map result_obj = set_snark_worker_graphql () in - let returned_last_snark_worker_opt = - result_obj.setSnarkWorker.lastSnarkWorker - in - let last_snark_worker = - match returned_last_snark_worker_opt with - | None -> - "" - | Some last -> - last |> Scalars.PublicKey.serialize |> Yojson.Basic.to_string - in - [%log info] "snark worker changed, lastSnarkWorker: %s" last_snark_worker - ~metadata:[ ("lastSnarkWorker", `String last_snark_worker) ] ; - () - - let must_set_snark_worker ~logger t ~new_snark_pub_key = - set_snark_worker ~logger t ~new_snark_pub_key - |> Deferred.bind ~f:Malleable_error.or_hard_error - - let start_filtered_log ~logger ~log_filter t = - let open Deferred.Let_syntax in - let query_obj = - Graphql.StartFilteredLog.(make @@ makeVariables ~filter:log_filter ()) - in - let%bind res = - exec_graphql_request ~logger:(Logger.null ()) ~retry_delay_sec:10.0 - ~node:t ~query_name:"StartFilteredLog" query_obj - in - match res with - | Ok query_result_obj -> - if query_result_obj.startFilteredLog then ( - [%log trace] "%s's log filter started" @@ id t ; - return (Ok ()) ) - else ( - [%log error] - "Attempted to start structured log collection on $node, but it had \ - already started" - ~metadata:[ ("node", `String t.node_id) ] ; - return (Ok ()) ) - | Error e -> - [%log error] "Error encountered while starting $node's log filter: $err" - ~metadata: - [ ("node", `String (id t)) - ; ("err", `String (Error.to_string_hum e)) - ] ; - return (Error e) - - let get_filtered_log_entries ~last_log_index_seen t = - let open Deferred.Or_error.Let_syntax in - let query_obj = - Graphql.GetFilteredLogEntries.( - make @@ makeVariables ~offset:last_log_index_seen ()) - in - let%bind query_result_obj = - exec_graphql_request ~logger:(Logger.null ()) ~retry_delay_sec:10.0 - ~node:t ~query_name:"GetFilteredLogEntries" query_obj - in - let log_entries = query_result_obj.getFilteredLogEntries in - if log_entries.isCapturing then return log_entries.logMessages - else - Deferred.Or_error.error_string - "Node is not currently capturing structured log messages" - - let get_metrics ~logger t = - let open Deferred.Or_error.Let_syntax in - [%log info] "Getting node's metrics" ~metadata:(logger_metadata t) ; - let query_obj = Graphql.Query_metrics.make () in - let%bind query_result_obj = - exec_graphql_request ~logger ~node:t ~query_name:"query_metrics" query_obj - in - [%log info] "get_metrics, finished exec_graphql_request" ; - let block_production_delay = - Array.to_list - @@ query_result_obj.daemonStatus.metrics.blockProductionDelay - in - let metrics = query_result_obj.daemonStatus.metrics in - let transaction_pool_diff_received = metrics.transactionPoolDiffReceived in - let transaction_pool_diff_broadcasted = - metrics.transactionPoolDiffBroadcasted - in - let transactions_added_to_pool = metrics.transactionsAddedToPool in - let transaction_pool_size = metrics.transactionPoolSize in - [%log info] - "get_metrics, result of graphql query (block_production_delay; \ - tx_received; tx_broadcasted; txs_added_to_pool; tx_pool_size) (%s; %d; \ - %d; %d; %d)" - ( String.concat ~sep:", " - @@ List.map ~f:string_of_int block_production_delay ) - transaction_pool_diff_received transaction_pool_diff_broadcasted - transactions_added_to_pool transaction_pool_size ; - return - Intf. - { block_production_delay - ; transaction_pool_diff_broadcasted - ; transaction_pool_diff_received - ; transactions_added_to_pool - ; transaction_pool_size - } - let dump_archive_data ~logger (t : t) ~data_file = let args = [ ("network_id", `String t.network_id); ("node_id", `String t.node_id) ] @@ -1101,7 +136,8 @@ module Node = struct @@ sprintf "Node %s of type %s cannot dump archive data" t.node_id node_type - let run_replayer ~logger (t : t) = + let run_replayer ?(start_slot_since_genesis = 0) ~logger (t : t) = + let open Malleable_error.Let_syntax in [%log info] "Running replayer on archived data node: %s" t.node_id ; let args = [ ("network_id", `String t.network_id) @@ -1112,7 +148,8 @@ module Node = struct try let%bind replay = match%map - Config_file.run_command ~config:!config_path ~args "run_replayer" + Deferred.bind ~f:Malleable_error.return + @@ Config_file.run_command ~config:!config_path ~args "run_replayer" with | Ok output -> Yojson.Safe.from_string output diff --git a/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml b/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml index 3f56c30febf..e59e0a04fbd 100644 --- a/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml +++ b/src/lib/integration_test_abstract_engine/graphql_polling_log_engine.ml @@ -60,7 +60,10 @@ let rec poll_get_filtered_log_entries_node ~logger ~event_writer let open Deferred.Let_syntax in if not (Pipe.is_closed event_writer) then ( let%bind () = after (Time.Span.of_ms 10000.0) in - match%bind Node.get_filtered_log_entries ~last_log_index_seen node with + match%bind + Graphql_requests.get_filtered_log_entries ~last_log_index_seen + @@ Node.get_ingress_uri node + with | Ok log_entries -> Array.iter log_entries ~f:(fun log_entry -> match parse_event_from_log_entry ~logger log_entry with @@ -91,7 +94,10 @@ let rec poll_get_filtered_log_entries_node ~logger ~event_writer let rec poll_start_filtered_log_node ~logger ~log_filter ~event_writer node = let open Deferred.Let_syntax in if not (Pipe.is_closed event_writer) then - match%bind Node.start_filtered_log ~logger ~log_filter node with + match%bind + Graphql_requests.start_filtered_log ~logger ~log_filter + @@ Node.get_ingress_uri node + with | Ok () -> return (Ok ()) | Error _ -> diff --git a/src/lib/integration_test_abstract_engine/network_config.ml b/src/lib/integration_test_abstract_engine/network_config.ml index f005479ed63..5a4e4e01c6d 100644 --- a/src/lib/integration_test_abstract_engine/network_config.ml +++ b/src/lib/integration_test_abstract_engine/network_config.ml @@ -94,6 +94,7 @@ let expand ~logger ~test_name ~(cli_inputs : Cli_inputs.t) ~(debug : bool) ; slots_per_sub_window ; txpool_max_size ; seed_nodes + ; _ } = test_config in diff --git a/src/lib/integration_test_cloud_engine/mina_automation.ml b/src/lib/integration_test_cloud_engine/mina_automation.ml index 084dd2f580c..5acad4c2290 100644 --- a/src/lib/integration_test_cloud_engine/mina_automation.ml +++ b/src/lib/integration_test_cloud_engine/mina_automation.ml @@ -179,7 +179,7 @@ module Network_config = struct let add_accounts accounts_and_keypairs = List.map accounts_and_keypairs ~f:(fun - ( { Test_config.Test_Account.balance; account_name; timing } + ( { Test_config.Test_Account.balance; account_name; timing; _ } , (pk, sk) ) -> let timing = runtime_timing_of_timing timing in @@ -267,7 +267,7 @@ module Network_config = struct (epoch_accounts : Test_config.Test_Account.t list) = let epoch_ledger_accounts = List.map epoch_accounts - ~f:(fun { account_name; balance; timing } -> + ~f:(fun { account_name; balance; timing; _ } -> let balance = Balance.of_mina_string_exn balance in let timing = runtime_timing_of_timing timing in let genesis_account = diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index b8a38157342..f355f7e0e66 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -47,8 +47,6 @@ module Engine = struct val id : t -> string - val app_id : t -> string - val network_keypair : t -> Network_keypair.t option val start : fresh_state:bool -> t -> unit Malleable_error.t @@ -58,35 +56,8 @@ module Engine = struct (** Returns true when [start] was most recently called, or false if [stop] was more recent. *) - val should_be_running : t -> bool -<<<<<<< HEAD val get_ingress_uri : t -> Uri.t -======= - val get_pooled_zkapp_commands : - logger:Logger.t - -> t - -> pk:Signature_lib.Public_key.Compressed.t - -> string list Deferred.Or_error.t - - val get_peer_id : - logger:Logger.t -> t -> (string * string list) Deferred.Or_error.t - - val must_get_peer_id : - logger:Logger.t -> t -> (string * string list) Malleable_error.t - - val get_best_chain : - ?max_length:int - -> logger:Logger.t - -> t - -> best_chain_block list Deferred.Or_error.t - - val must_get_best_chain : - ?max_length:int - -> logger:Logger.t - -> t - -> best_chain_block list Malleable_error.t ->>>>>>> 6131f41e37 ((intg test) Undo function renaming) val dump_archive_data : logger:Logger.t -> t -> data_file:string -> unit Malleable_error.t diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 7c70d1017a2..974793920af 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -67,9 +67,10 @@ module Epoch_data = struct module Data = struct (* the seed is a field value in Base58Check format *) type t = { epoch_ledger : Test_Account.t list; epoch_seed : string } + [@@deriving to_yojson] end - type t = { staking : Data.t; next : Data.t option } + type t = { staking : Data.t; next : Data.t option } [@@deriving to_yojson] end module Git_build = struct