feat(permissionless batches): batch production toolkit and operator recovery

build/dockerfiles/coordinator-api.Dockerfile.dockerignore

@@ -4,3 +4,5 @@ docs/
 l2geth/
 rpc-gateway/
 *target/*
+
+permissionless-batches/conf/

build/dockerfiles/coordinator-cron.Dockerfile.dockerignore

@@ -4,3 +4,5 @@ docs/
 l2geth/
 rpc-gateway/
 *target/*
+
+permissionless-batches/conf/

build/dockerfiles/db_cli.Dockerfile.dockerignore

@@ -4,3 +4,5 @@ docs/
 l2geth/
 rpc-gateway/
 *target/*
+
+permissionless-batches/conf/

build/dockerfiles/gas_oracle.Dockerfile.dockerignore

@@ -1,5 +1,8 @@
 assets/
+contracts/
 docs/
 l2geth/
 rpc-gateway/
-*target/*
+*target/*
+
+permissionless-batches/conf/

build/dockerfiles/recovery_permissionless_batches.Dockerfile

@@ -0,0 +1,30 @@
+# Download Go dependencies
+FROM scrolltech/go-rust-builder:go-1.21-rust-nightly-2023-12-03 as base
+
+WORKDIR /src
+COPY go.work* ./
+COPY ./rollup/go.* ./rollup/
+COPY ./common/go.* ./common/
+COPY ./coordinator/go.* ./coordinator/
+COPY ./database/go.* ./database/
+COPY ./tests/integration-test/go.* ./tests/integration-test/
+COPY ./bridge-history-api/go.* ./bridge-history-api/
+RUN go mod download -x
+
+# Build rollup_relayer
+FROM base as builder
+
+RUN --mount=target=. \
+    --mount=type=cache,target=/root/.cache/go-build \
+    cd /src/rollup/cmd/permissionless_batches/ && CGO_LDFLAGS="-ldl" go build -v -p 4 -o /bin/rollup_relayer
+
+# Pull rollup_relayer into a second stage deploy ubuntu container
+FROM ubuntu:20.04
+
+RUN apt update && apt install vim netcat-openbsd net-tools curl ca-certificates -y
+
+ENV CGO_LDFLAGS="-ldl"
+
+COPY --from=builder /bin/rollup_relayer /bin/
+WORKDIR /app
+ENTRYPOINT ["rollup_relayer"]

build/dockerfiles/recovery_permissionless_batches.Dockerfile.dockerignore

@@ -0,0 +1,8 @@
+assets/
+contracts/
+docs/
+l2geth/
+rpc-gateway/
+*target/*
+
+permissionless-batches/conf/

build/dockerfiles/rollup_relayer.Dockerfile.dockerignore

@@ -1,5 +1,8 @@
 assets/
+contracts/
 docs/
 l2geth/
 rpc-gateway/
-*target/*
+*target/*
+
+permissionless-batches/conf/

go.work

@@ -3,10 +3,11 @@ go 1.22
 toolchain go1.22.2
 
 use (
-	./bridge-history-api
+	//./bridge-history-api
 	./common
 	./coordinator
 	./database
 	./rollup
 	./tests/integration-test
+	//../go-ethereum
 )

go.work.sum

@@ -718,8 +718,10 @@ github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/compose-spec/compose-go v1.20.0 h1:h4ZKOst1EF/DwZp7dWkb+wbTVE4nEyT9Lc89to84Ol4=
 github.com/compose-spec/compose-go v1.20.0/go.mod h1:+MdqXV4RA7wdFsahh/Kb8U0pAJqkg7mr4PM9tFKU8RM=
+github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
 github.com/consensys/bavard v0.1.27/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs=
 github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY=
+github.com/consensys/gnark-crypto v0.13.0/go.mod h1:wKqwsieaKPThcFkHe0d0zMsbHEUWFmZcG7KBCse210o=
 github.com/container-orchestrated-devices/container-device-interface v0.6.1 h1:mz77uJoP8im/4Zins+mPqt677ZMaflhoGaYrRAl5jvA=
 github.com/container-orchestrated-devices/container-device-interface v0.6.1/go.mod h1:40T6oW59rFrL/ksiSs7q45GzjGlbvxnA4xaK6cyq+kA=
 github.com/containerd/aufs v1.0.0 h1:2oeJiwX5HstO7shSrPZjrohJZLzK36wvpdmzDRkL/LY=
@@ -1167,6 +1169,7 @@ github.com/labstack/echo/v4 v4.9.0/go.mod h1:xkCDAdFCIf8jsFQ5NnbK7oqaF/yU1A1X20L
 github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/labstack/gommon v0.3.1/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
+github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
 github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A=
 github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
 github.com/lestrrat-go/blackmagic v1.0.0 h1:XzdxDbuQTz0RZZEmdU7cnQxUtFUzgCSPq8RCz4BxIi4=
@@ -1357,7 +1360,7 @@ github.com/scroll-tech/da-codec v0.1.1-0.20241014152913-2703f226fb0b h1:5H6V6yba
 github.com/scroll-tech/da-codec v0.1.1-0.20241014152913-2703f226fb0b/go.mod h1:48uxaqVgpD8ulH8p+nrBtfeLHZ9tX82bVVdPNkW3rPE=
 github.com/scroll-tech/da-codec v0.1.3-0.20250227072756-a1482833595f h1:YYbhuUwjowqI4oyXtECRofck7Fyj18e1tcRjuQlZpJE=
 github.com/scroll-tech/da-codec v0.1.3-0.20250227072756-a1482833595f/go.mod h1:xECEHZLVzbdUn+tNbRJhRIjLGTOTmnFQuTgUTeVLX58=
-github.com/scroll-tech/da-codec v0.1.3-0.20250519114140-bfa7133d4ad1/go.mod h1:yhTS9OVC0xQGhg7DN5iV5KZJvnSIlFWAxDdp+6jxQtY=
+github.com/scroll-tech/da-codec v0.1.3-0.20250313120912-344f2d5e33e1/go.mod h1:yhTS9OVC0xQGhg7DN5iV5KZJvnSIlFWAxDdp+6jxQtY=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20240607130425-e2becce6a1a4/go.mod h1:byf/mZ8jLYUCnUePTicjJWn+RvKdxDn7buS6glTnMwQ=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20240821074444-b3fa00861e5e/go.mod h1:swB5NSp8pKNDuYsTxfR08bHS6L56i119PBx8fxvV8Cs=
 github.com/scroll-tech/go-ethereum v1.10.14-0.20241010064814-3d88e870ae22/go.mod h1:r9FwtxCtybMkTbWYCyBuevT9TW3zHmOTHqD082Uh+Oo=

permissionless-batches/Makefile

@@ -0,0 +1,25 @@
+.PHONY: batch-production-submission launch_local_prover launch_cloud_prover psql check_proving_status
+
+export SCROLL_ZKVM_VERSION=0.3.0
+PG_URL=postgres://postgres@localhost:5432/scroll
+
+batch_production_submission:
+	docker compose --profile batch-production-submission up
+
+local_prover:
+	docker compose --profile local-prover up -d
+
+cloud_prover:
+	docker compose --profile cloud-prover up
+
+psql:
+	psql 'postgres://postgres@localhost:5432/scroll'
+
+check_proving_status:
+	@echo "Checking proving status..."
+	@result=$$(psql "${PG_URL}" -t -c "SELECT proving_status = 4 AS is_status_success FROM batch ORDER BY index LIMIT 1;" | tr -d '[:space:]'); \
+	if [ "$$result" = "t" ]; then \
+		echo "✅ Prove succeeded! You're ready to submit permissionless batch and proof!"; \
+	else \
+		echo "Proof is not ready..."; \
+	fi

permissionless-batches/README.md

@@ -0,0 +1,186 @@
+# Permissionless Batches
+Permissionless batches aka enforced batches is a feature that provides guarantee to users that they can exit Scroll even if the operator is down or censoring. 
+It allows anyone to take over and submit a batch (permissionless batch submission) together with a proof after a certain time period has passed without a batch being finalized on L1. 
+
+Once permissionless batch mode is activated, the operator can no longer submit batches in a permissioned way. Only the security council can deactivate permissionless batch mode and reinstate the operator as the only batch submitter.
+There are two types of situations to consider:
+- `Permissionless batch mode is activated:` This means that finalization halted for some time. Now anyone can submit batches utilizing the [batch production toolkit](#batch-production-toolkit).
+- `Permissionless batch mode is deactivated:` This means that the security council has decided to reinstate the operator as the only batch submitter. The operator needs to [recover](#operator-recovery) the sequencer and relayer to resume batch submission and the valid L2 chain.
+
+
+## Batch production toolkit
+The batch production toolkit is a set of tools that allow anyone to submit a batch in permissionless mode. It consists of three main components:
+1. l2geth state recovery from L1
+2. l2geth block production
+3. production, proving and submission of batch with `docker-compose.yml`
+
+### Prerequisites
+- Unix-like OS, 32GB RAM
+- Docker
+- [l2geth](https://github.com/scroll-tech/go-ethereum/) or [Docker image](https://hub.docker.com/r/scrolltech/l2geth) of corresponding version [TODO link list with versions](#batch-production-toolkit).
+- access to an Ethereum L1 RPC node (beacon node and execution client)
+- ability to run a prover or access to a proving service (e.g. Sindri)
+- L1 account with funds to pay for the batch submission
+
+### 1. l2geth state recovery from L1
+Once permissionless mode is activated there's no blocks being produced and propagated on L2. The first step is to recover the latest state of the L2 chain from L1. This is done by running l2geth in recovery mode. 
+More information about l2geth recovery (aka L1 follower mode) can be found [here TODO: put correct link once released](https://github.com/scroll-tech/scroll-documentation/pull/374).
+
+Running l2geth in recovery mode requires following configuration:
+- `--scroll` or `--scroll-sepolia` - enables Scroll Mainnet or Sepolia mode
+- `--da.blob.beaconnode` - L1 RPC beacon node
+- `--l1.endpoint` - L1 RPC execution client
+- `--da.sync=true` - enables syncing with L1
+- `--da.recovery` - enables recovery mode
+- `--da.recovery.initiall1block` - initial L1 block (commit tx of initial batch)
+- `--da.recovery.initialbatch` - batch where to start recovery from. Can be found on [Scrollscan Explorer](https://scrollscan.com/batches).
+- `--da.recovery.l2endblock` - until which L2 block recovery should run (optional)
+
+```bash
+./build/bin/geth --scroll<-sepolia> \
+--datadir "tmp/datadir" \
+--gcmode archive \
+--http --http.addr "0.0.0.0" --http.port 8545 --http.api "eth,net,web3,debug,scroll" --http.vhosts "*" \
+--da.blob.beaconnode "<L1 RPC beacon node>" \
+--l1.endpoint "<L1 RPC execution client>" \
+--da.sync=true --da.recovery --da.recovery.initiall1block "<initial L1 block (commit tx of initial batch)>" --da.recovery.initialbatch "<batch where to start recovery from>" --da.recovery.l2endblock "<until which L2 block recovery should run (optional)>" \
+--verbosity 3
+```
+
+### 2. l2geth block production
+After the state is recovered, the next step is to produce blocks on L2. This is done by running l2geth in block production mode.
+As a prerequisite, the state recovery must be completed and the latest state of the L2 chain must be available.
+
+You also need to generate a keystore e.g. with [Clef](https://geth.ethereum.org/docs/fundamentals/account-management) to be able to sign blocks. 
+This key is not used for any funds, but required for block production to work. Once you generated blocks you can safely discard it.
+
+Running l2geth in block production mode requires following configuration:
+- `--scroll` or `--scroll-sepolia` - enables Scroll Mainnet or Sepolia mode
+- `--da.blob.beaconnode` - L1 RPC beacon node
+- `--l1.endpoint` - L1 RPC execution client
+- `--da.sync=true` - enables syncing with L1
+- `--da.recovery` - enables recovery mode
+- `--da.recovery.produceblocks` - enables block production
+- `--miner.etherbase '0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee' --mine` - enables mining. the address is not used, but required for mining to work
+- `---miner.gaslimit 1 --miner.gasprice 1 --miner.maxaccountsnum 100 --rpc.gascap 0 --gpo.ignoreprice 1` - gas limits for block production
+
+```bash
+./build/bin/geth --scroll<-sepolia> \
+--datadir "tmp/datadir" \
+--gcmode archive \
+--http --http.addr "0.0.0.0" --http.port 8545 --http.api "eth,net,web3,debug,scroll" --http.vhosts "*" \
+--da.blob.beaconnode "<L1 RPC beacon node>" \
+--l1.endpoint "<L1 RPC execution client>" \
+--da.sync=true --da.recovery --da.recovery.produceblocks \
+--miner.gaslimit 1 --miner.gasprice 1 --miner.maxaccountsnum 100 --rpc.gascap 0 --gpo.ignoreprice 1 \
+--miner.etherbase '0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee' --mine \
+--ccc \
+--verbosity 3
+```
+
+### 3. production, proving and submission of batch with `docker-compose.yml`
+After the blocks are produced, the next step is to produce a batch, prove it and submit it to L1. This is done by running the `docker-compose.yml` in the `permissionless-batches` folder.
+
+
+#### Producing a batch
+To produce a batch you need to run the `batch-production-submission` profile in `docker-compose.yml`.
+
+1. Fill `conf/genesis.json` with the latest genesis state from the L2 chain. The genesis for the current fork can be found here: [TODO link list with versions](#batch-production-toolkit)
+2. Make sure that `l2geth` with your locally produced blocks is running and reachable from the Docker network (e.g. `http://host.docker.internal:8545`)
+3. Fill in required fields in `conf/relayer/config.json`
+
+
+Run with `make batch_production_submission`.
+This will produce chunks, a batch and bundle which will be proven in the next step.
+`Success! You're ready to generate proofs!` indicates that everything is working correctly and the batch is ready to be proven.
+
+#### Proving a batch
+To prove the chunk, batch and bundle you just generated you need to run the `local-prover` or `cloud-prover` profile in `docker-compose.yml`.
+
+Local Proving:
+
+1. Hardware spec for local prover: CPU: 36+ core, 128G memory GPU: 24G memory (eg. Rtx 3090/3090Ti/4090/A10/L4)
+2. Make sure `verifier` `low_version_circuit` and `high_version_circuit` in `conf/coordinator/config.json` are correct for the latest fork: [TODO link list with versions](#batch-production-toolkit)
+2. Set the `SCROLL_ZKVM_VERSION` environment variable on `Makefile` to the correct version. [TODO link list with versions](#batch-production-toolkit)
+4. Fill in the required fields in `conf/proving-service/local-prover/config.json`
+
+Run with `make local_prover`.
+
+Cloud Proving(not supported yet):
+
+1. Make sure `verifier` `low_version_circuit` and `high_version_circuit` in `conf/coordinator/config.json` are correct for the latest fork: [TODO link list with versions](#batch-production-toolkit)
+2. Set the `SCROLL_ZKVM_VERSION` environment variable on `Makefile` to the correct version. [TODO link list with versions](#batch-production-toolkit)
+3. Fill in the required fields in `conf/proving-service/cloud-prover/config.json`. It is recommended to use Sindri. You'll need to obtain credits and an API key from their [website](https://sindri.app/).
+
+Run with `make cloud_prover`.
+
+This will prove chunks, the batch and bundle.
+Run `make check_proving_status`
+`Success! You're ready to submit permissionless batch and proof!` indicates that everything is working correctly and the batch is ready to be submit.
+
+
+#### Batch submission
+To submit the batch you need to run the `batch-production-submission` profile in `docker-compose.yml`.
+
+1. Fill in required fields in `conf/relayer/config.json` for the sender config.
+
+Run with `make batch_production_submission`.
+This will submit the batch to L1 and finalize it. The transaction will be retried in case of failure.
+
+**Troubleshooting**
+- in case the submission fails it will print the calldata for the transaction in an error message. You can use this with `cast call --trace --rpc-url "$SCROLL_L1_DEPLOYMENT_RPC" "$L1_SCROLL_CHAIN_PROXY_ADDR" <calldata>` to see what went wrong.
+  - `0x4df567b9: ErrorNotInEnforcedBatchMode`: permissionless batch mode is not activated, you can't submit a batch
+  - `0xa5d305cc: ErrorBatchIsEmpty`: no blob was provided. This is usually returned if you do the `cast call`, permissionless mode is activated but you didn't provide a blob in the transaction.
+
+## Operator recovery
+Operator recovery needs to be run by the rollup operator to resume normal rollup operation after permissionless batch mode is deactivated. It consists of two main components:
+1. l2geth recovery
+2. Relayer recovery
+
+These steps are required to resume permissioned batch submission and the valid L2 chain. They will restore the entire history of the batches submitted during permissionless mode. 
+
+### Prerequisites
+- l2geth with the latest state of the L2 chain (before permissionless mode was activated)
+- signer key for the sequencer according to Clique consensus
+- relayer and coordinator are set up, running and up-to-date with the latest state of the L2 chain (before permissionless mode was activated)
+
+### l2geth recovery
+Running l2geth in recovery mode requires following configuration:
+- `--scroll` or `--scroll-sepolia` - enables Scroll Mainnet or Sepolia mode
+- `--da.blob.beaconnode` - L1 RPC beacon node
+- `--l1.endpoint` - L1 RPC execution client
+- `--da.sync=true` - enables syncing with L1
+- `--da.recovery` - enables recovery mode
+- `--da.recovery.signblocks` - enables signing blocks with the sequencer and configured key
+- `--da.recovery.initiall1block` - initial L1 block (commit tx of initial batch)
+- `--da.recovery.initialbatch` - batch where to start recovery from. Can be found on [Scrollscan Explorer](https://scrollscan.com/batches).
+- `--da.recovery.l2endblock` - until which L2 block recovery should run (optional)
+
+```bash
+./build/bin/geth --scroll<-sepolia> \
+--datadir "tmp/datadir" \
+--gcmode archive \
+--http --http.addr "0.0.0.0" --http.port 8545 --http.api "eth,net,web3,debug,scroll" --http.vhosts "*" \
+--da.blob.beaconnode "<L1 RPC beacon node>" \
+--l1.endpoint "<L1 RPC execution client>" \
+--da.sync=true --da.recovery --da.recovery.signblocks --da.recovery.initiall1block "<initial L1 block (commit tx of initial batch)>" --da.recovery.initialbatch "<batch where to start recovery from>" --da.recovery.l2endblock "<until which L2 block recovery should run (optional)>" \
+--verbosity 3
+```
+
+After the recovery is finished, start the sequencer in normal operation and continue issuing L2 blocks as normal. This will resume the L2 chain, allow the relayer (after running recovery) to create new batches and allow other L2 follower nodes to sync up the valid and signed L2 chain.
+
+### Relayer recovery
+Start the relayer with the following additional top-level configuration:
+```
+  "recovery_config": {
+    "enable": true
+  }
+```
+
+This will make the relayer recover all the chunks, batches and bundles that were submitted during permissionless mode. These batches are marked automatically as proven and finalized.
+Once this process is finished, start the relayer normally without the recovery config to resume normal operation.
+```
+  "recovery_config": {
+    "enable": false
+  }
+```

permissionless-batches/conf/coordinator/config.json

@@ -0,0 +1,37 @@
+{
+  "prover_manager": {
+    "provers_per_session": 1,
+    "session_attempts": 100,
+    "chunk_collection_time_sec": 36000,
+    "batch_collection_time_sec": 2700,
+    "bundle_collection_time_sec": 2700,
+    "verifier": {
+      "mock_mode": false,
+      "low_version_circuit" : {
+        "fork_name": "darwinV2",
+        "params_path": "/verifier/params",
+        "assets_path": "/verifier/assets",
+        "min_prover_version": "v4.4.56"
+      },
+      "high_version_circuit" : {
+        "fork_name": "euclid",
+        "assets_path": "/verifier/openvm/verifier",
+        "min_prover_version": "v4.5.7"
+      }
+    }
+  },
+  "db": {
+    "driver_name": "postgres",
+    "dsn": "postgres://db/scroll?sslmode=disable&user=postgres",
+    "maxOpenNum": 200,
+    "maxIdleNum": 20
+  },
+  "l2": {
+    "chain_id": 333333
+  },
+  "auth": {
+    "secret": "e788b62d39254928a821ac1c76b274a8c835aa1e20ecfb6f50eb10e87847de44",
+    "challenge_expire_duration_sec": 10,
+    "login_expire_duration_sec": 3600
+  }
+}