feat: add additional configuration options for participants (#81)

.github/tests/hildr.yaml

@@ -9,3 +9,7 @@ optimism_package:
         cl_type: hildr
       - el_type: op-erigon
         cl_type: hildr
+      - el_type: op-nethermind
+        cl_type: hildr
+      - el_type: op-besu
+        cl_type: hildr

.github/tests/op-besu.yaml

@@ -3,3 +3,5 @@ optimism_package:
     - participants:
         - el_type: op-besu
           cl_type: op-node
+        - el_type: op-besu
+          cl_type: hildr

.github/tests/op-erigon.yaml

@@ -3,3 +3,5 @@ optimism_package:
     - participants:
         - el_type: op-erigon
           cl_type: op-node
+        - el_type: op-erigon
+          cl_type: hildr

.github/tests/op-geth.yaml

@@ -3,3 +3,5 @@ optimism_package:
     - participants:
         - el_type: op-geth
           cl_type: op-node
+        - el_type: op-geth
+          cl_type: hildr

.github/tests/op-nethermind.yaml

@@ -4,3 +4,5 @@ optimism_package:
         - el_type: op-geth
         - el_type: op-nethermind
           cl_type: op-node
+        - el_type: op-nethermind
+          cl_type: hildr

.github/tests/op-node.yaml

@@ -9,3 +9,5 @@ optimism_package:
           cl_type: op-node
         - el_type: op-nethermind
           cl_type: op-node
+        - el_type: op-besu
+          cl_type: op-node

.github/tests/op-reth.yaml

@@ -3,3 +3,5 @@ optimism_package:
     - participants:
         - el_type: op-reth
           cl_type: op-node
+        - el_type: op-reth
+          cl_type: hildr

README.md

@@ -73,6 +73,48 @@ optimism_package:
         # - op-besu: ghcr.io/optimism-java/op-besu:latest
         el_image: ""
 
+        # The log level string that this participant's EL client should log at
+        # If this is emptystring then the global `logLevel` parameter's value will be translated into a string appropriate for the client (e.g. if
+        # global `logLevel` = `info` then Geth would receive `3`, Besu would receive `INFO`, etc.)
+        # If this is not emptystring, then this value will override the global `logLevel` setting to allow for fine-grained control
+        # over a specific participant's logging
+        el_log_level: ""
+
+        # A list of optional extra env_vars the el container should spin up with
+        el_extra_env_vars: {}
+
+        # A list of optional extra labels the el container should spin up with
+        # Example; el_extra_labels: {"ethereum-package.partition": "1"}
+        el_extra_labels: {}
+
+        # A list of optional extra params that will be passed to the EL client container for modifying its behaviour
+        el_extra_params: []
+
+        # A list of tolerations that will be passed to the EL client container
+        # Only works with Kubernetes
+        # Example: el_tolerations:
+        # - key: "key"
+        #   operator: "Equal"
+        #   value: "value"
+        #   effect: "NoSchedule"
+        #   toleration_seconds: 3600
+        # Defaults to empty
+        el_tolerations: []
+
+        # Persistent storage size for the EL client container (in MB)
+        # Defaults to 0, which means that the default size for the client will be used
+        # Default values can be found in /src/package_io/constants.star VOLUME_SIZE
+        el_volume_size: 0
+
+        # Resource management for el containers
+        # CPU is milicores
+        # RAM is in MB
+        # Defaults to 0, which results in no resource limits
+        el_min_cpu: 0
+        el_max_cpu: 0
+        el_min_mem: 0
+        el_max_mem: 0
+
       # CL(Consensus Layer) Specific flags
         # The type of CL client that should be started
         # Valid values are:
@@ -86,6 +128,67 @@ optimism_package:
         # - hildr: ghcr.io/optimism-java/hildr:latest
         cl_image: ""
 
+        # The log level string that this participant's CL client should log at
+        # If this is emptystring then the global `logLevel` parameter's value will be translated into a string appropriate for the client (e.g. if
+        # If this is not emptystring, then this value will override the global `logLevel` setting to allow for fine-grained control
+        # over a specific participant's logging
+        cl_log_level: ""
+
+        # A list of optional extra env_vars the cl container should spin up with
+        cl_extra_env_vars: {}
+
+        # A list of optional extra labels that will be passed to the CL client Beacon container.
+        # Example; cl_extra_labels: {"ethereum-package.partition": "1"}
+        cl_extra_labels: {}
+
+        # A list of optional extra params that will be passed to the CL client Beacon container for modifying its behaviour
+        # If the client combines the Beacon & validator nodes (e.g. Teku, Nimbus), then this list will be passed to the combined Beacon-validator node
+        cl_extra_params: []
+
+        # A list of tolerations that will be passed to the CL client container
+        # Only works with Kubernetes
+        # Example: el_tolerations:
+        # - key: "key"
+        #   operator: "Equal"
+        #   value: "value"
+        #   effect: "NoSchedule"
+        #   toleration_seconds: 3600
+        # Defaults to empty
+        cl_tolerations: []
+
+        # Persistent storage size for the CL client container (in MB)
+        # Defaults to 0, which means that the default size for the client will be used
+        # Default values can be found in /src/package_io/constants.star VOLUME_SIZE
+        cl_volume_size: 0
+
+        # Resource management for cl containers
+        # CPU is milicores
+        # RAM is in MB
+        # Defaults to 0, which results in no resource limits
+        cl_min_cpu: 0
+        cl_max_cpu: 0
+        cl_min_mem: 0
+        cl_max_mem: 0
+
+        # Participant specific flags
+        # Node selector
+        # Only works with Kubernetes
+        # Example: node_selectors: { "disktype": "ssd" }
+        # Defaults to empty
+        node_selectors: {}
+
+        # A list of tolerations that will be passed to the EL/CL/validator containers
+        # This is to be used when you don't want to specify the tolerations for each container separately
+        # Only works with Kubernetes
+        # Example: tolerations:
+        # - key: "key"
+        #   operator: "Equal"
+        #   value: "value"
+        #   effect: "NoSchedule"
+        #   toleration_seconds: 3600
+        # Defaults to empty
+        tolerations: []
+
         # Count of nodes to spin up for this participant
         # Default to 1
         count: 1
@@ -142,6 +245,32 @@ optimism_package:
   op_contract_deployer_params:
     image: mslipper/op-deployer:latest
     artifacts_url: https://storage.googleapis.com/oplabs-contract-artifacts/artifacts-v1-4accd01f0c35c26f24d2aa71aba898dd7e5085a2ce5daadc8a84b10caf113409.tar.gz
+
+  # The global log level that all clients should log at
+  # Valid values are "error", "warn", "info", "debug", and "trace"
+  # This value will be overridden by participant-specific values
+  global_log_level: "info"
+
+  # Global node selector that will be passed to all containers (unless overridden by a more specific node selector)
+  # Only works with Kubernetes
+  # Example: global_node_selectors: { "disktype": "ssd" }
+  # Defaults to empty
+  global_node_selectors: {}
+
+  # Global tolerations that will be passed to all containers (unless overridden by a more specific toleration)
+  # Only works with Kubernetes
+  # Example: tolerations:
+  # - key: "key"
+  #   operator: "Equal"
+  #   value: "value"
+  #   effect: "NoSchedule"
+  #   toleration_seconds: 3600
+  # Defaults to empty
+  global_tolerations: []
+
+  # Whether the environment should be persistent; this is WIP and is slowly being rolled out accross services
+  # Defaults to false
+  persistent: false
 ```
 
 ### Additional configuration recommendations

main.star

@@ -1,8 +1,5 @@
 ethereum_package = import_module("github.com/ethpandaops/ethereum-package/main.star")
 contract_deployer = import_module("./src/contracts/contract_deployer.star")
-static_files = import_module(
-    "github.com/ethpandaops/ethereum-package/src/static_files/static_files.star"
-)
 l2_launcher = import_module("./src/l2.star")
 wait_for_sync = import_module("./src/wait/wait_for_sync.star")
 input_parser = import_module("./src/package_io/input_parser.star")
@@ -26,6 +23,11 @@ def run(plan, args):
     # .get will return None if the key is in the config with a None value.
     optimism_args = args.get("optimism_package") or input_parser.default_optimism_args()
     optimism_args_with_right_defaults = input_parser.input_parser(plan, optimism_args)
+    global_tolerations = optimism_args_with_right_defaults.global_tolerations
+    global_node_selectors = optimism_args_with_right_defaults.global_node_selectors
+    global_log_level = optimism_args_with_right_defaults.global_log_level
+    persistent = optimism_args_with_right_defaults.persistent
+
     # Deploy the L1
     plan.print("Deploying a local L1")
     l1 = ethereum_package.run(plan, ethereum_args)
@@ -64,6 +66,10 @@ def run(plan, args):
             l1_config_env_vars,
             l1_priv_key,
             all_l1_participants[0].el_context,
+            global_log_level,
+            global_node_selectors,
+            global_tolerations,
+            persistent,
         )
 
     return
@@ -78,6 +84,10 @@ def run(plan, args):
             l1_config_env_vars,
             l1_priv_key,
             all_l1_participants[0].el_context,
+            global_log_level,
+            global_node_selectors,
+            global_tolerations,
+            persistent,
         )
     elif type(optimism_args) == "list":
         seen_names = {}
@@ -108,6 +118,10 @@ def run(plan, args):
                 l1_config_env_vars,
                 l1_priv_key,
                 all_l1_participants[0].el_context,
+                global_log_level,
+                global_node_selectors,
+                global_tolerations,
+                persistent,
             )
     else:
         fail("invalid type provided for param: `optimism-package`")

network_params.yaml

@@ -3,8 +3,30 @@ optimism_package:
     - participants:
       - el_type: op-geth
         el_image: ""
+        el_log_level: ""
+        el_extra_env_vars: {}
+        el_extra_labels: {}
+        el_extra_params: []
+        el_tolerations: []
+        el_volume_size: 0
+        el_min_cpu: 0
+        el_max_cpu: 0
+        el_min_mem: 0
+        el_max_mem: 0
         cl_type: op-node
         cl_image: ""
+        cl_log_level: ""
+        cl_extra_env_vars: {}
+        cl_extra_labels: {}
+        cl_extra_params: []
+        cl_tolerations: []
+        cl_volume_size: 0
+        cl_min_cpu: 0
+        cl_max_cpu: 0
+        cl_min_mem: 0
+        cl_max_mem: 0
+        node_selectors: {}
+        tolerations: []
         count: 1
       network_params:
         network: "kurtosis"
@@ -19,3 +41,7 @@ optimism_package:
   op_contract_deployer_params:
     image: mslipper/op-deployer:latest
     artifacts_url: https://storage.googleapis.com/oplabs-contract-artifacts/artifacts-v1-4accd01f0c35c26f24d2aa71aba898dd7e5085a2ce5daadc8a84b10caf113409.tar.gz
+  global_log_level: "info"
+  global_node_selectors: {}
+  global_tolerations: []
+  persistent: false

src/batcher/op-batcher/op_batcher_launcher.star

@@ -1,8 +1,8 @@
-shared_utils = import_module(
+ethereum_package_shared_utils = import_module(
     "github.com/ethpandaops/ethereum-package/src/shared_utils/shared_utils.star"
 )
 
-constants = import_module(
+ethereum_package_constants = import_module(
     "github.com/ethpandaops/ethereum-package/src/package_io/constants.star"
 )
 
@@ -20,10 +20,10 @@ BATCHER_HTTP_PORT_NUM = 8548
 
 def get_used_ports():
     used_ports = {
-        BATCHER_HTTP_PORT_ID: shared_utils.new_port_spec(
+        BATCHER_HTTP_PORT_ID: ethereum_package_shared_utils.new_port_spec(
             BATCHER_HTTP_PORT_NUM,
-            shared_utils.TCP_PROTOCOL,
-            shared_utils.HTTP_APPLICATION_PROTOCOL,
+            ethereum_package_shared_utils.TCP_PROTOCOL,
+            ethereum_package_shared_utils.HTTP_APPLICATION_PROTOCOL,
         ),
     }
     return used_ports
@@ -95,5 +95,5 @@ def get_batcher_config(
         image=image,
         ports=ports,
         cmd=cmd,
-        private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
+        private_ip_address_placeholder=ethereum_package_constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
     )

src/blockscout/blockscout_launcher.star

@@ -1,16 +1,13 @@
-shared_utils = import_module(
+ethereum_package_shared_utils = import_module(
     "github.com/ethpandaops/ethereum-package/src/shared_utils/shared_utils.star"
 )
-constants = import_module(
-    "github.com/ethpandaops/ethereum-package/src/package_io/constants.star"
-)
 
 postgres = import_module("github.com/kurtosis-tech/postgres-package/main.star")
 
 util = import_module("../util.star")
 
-IMAGE_NAME_BLOCKSCOUT = "blockscout/blockscout-optimism:6.6.0"
-IMAGE_NAME_BLOCKSCOUT_VERIF = "ghcr.io/blockscout/smart-contract-verifier:v1.7.0"
+IMAGE_NAME_BLOCKSCOUT = "blockscout/blockscout-optimism:6.8.0"
+IMAGE_NAME_BLOCKSCOUT_VERIF = "ghcr.io/blockscout/smart-contract-verifier:v1.9.0"
 
 SERVICE_NAME_BLOCKSCOUT = "op-blockscout"
 
@@ -29,18 +26,18 @@ BLOCKSCOUT_VERIF_MIN_MEMORY = 10
 BLOCKSCOUT_VERIF_MAX_MEMORY = 1024
 
 USED_PORTS = {
-    HTTP_PORT_ID: shared_utils.new_port_spec(
+    HTTP_PORT_ID: ethereum_package_shared_utils.new_port_spec(
         HTTP_PORT_NUMBER,
-        shared_utils.TCP_PROTOCOL,
-        shared_utils.HTTP_APPLICATION_PROTOCOL,
+        ethereum_package_shared_utils.TCP_PROTOCOL,
+        ethereum_package_shared_utils.HTTP_APPLICATION_PROTOCOL,
     )
 }
 
 VERIF_USED_PORTS = {
-    HTTP_PORT_ID: shared_utils.new_port_spec(
+    HTTP_PORT_ID: ethereum_package_shared_utils.new_port_spec(
         HTTP_PORT_NUMBER_VERIF,
-        shared_utils.TCP_PROTOCOL,
-        shared_utils.HTTP_APPLICATION_PROTOCOL,
+        ethereum_package_shared_utils.TCP_PROTOCOL,
+        ethereum_package_shared_utils.HTTP_APPLICATION_PROTOCOL,
     )
 }
 
@@ -76,7 +73,7 @@ def launch_blockscout(
         SERVICE_NAME_BLOCKSCOUT, l2_services_suffix
     )
     verif_service = plan.add_service(verif_service_name, config_verif)
-    verif_url = "http://{}:{}/api".format(
+    verif_url = "http://{}:{}".format(
         verif_service.hostname, verif_service.ports["http"].number
     )