Support BSP export for EIP-4844 blob tx (Ethereum Migration v1.5)

.github/workflows/golangci-lint.yml

@@ -18,16 +18,17 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
-      - name: Set up Go 1.18
+      - name: Set up Go 1.22
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18.x
+          go-version: 1.22.0
         id: go
       - run: go version
 
       - name: Lint
         run: |
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.46.2
+          go get github.com/ethereum/go-ethereum/crypto/kzg4844
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.56.2
           ./bin/golangci-lint cache clean
           ./bin/golangci-lint run
 

.golangci.yml

@@ -13,16 +13,12 @@ run:
 linters:
   enable:
   #-------------main-list--------------#
-    - deadcode # Finds unused code
     - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases
     - gosimple # Linter for Go source code that specializes in simplifying a code
     - govet # Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string
     - ineffassign # Detects when assignments to existing variables are not used
-    - staticcheck # Staticcheck is a go vet on steroids, applying a ton of static analysis checks
-    - structcheck # Finds unused struct fields
     - typecheck # Like the front-end of a Go compiler, parses and type-checks Go code
     - unused # Checks Go code for unused constants, variables, functions and types
-    - varcheck # Finds unused global variables and constants
     - stylecheck # Stylecheck is a replacement for golint
     - gocritic # Provides diagnostics that check for bugs, performance and style issues. Extensible without recompilation through dynamic rules. Dynamic rules are written declaratively with AST patterns, filters, report message and optional suggestion.
     - gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification
@@ -32,7 +28,6 @@ linters:
     - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers
     - bidichk # Checks for dangerous unicode character sequences
     - bodyclose # Checks whether HTTP response body is closed successfully
-    - depguard # Go linter that checks if package imports are in a list of acceptable packages
     - dogsled # Checks assignments with too many blank identifiers (e.g. x, , , _, := f())
     - dupl # Tool for code clone detection
     - exhaustive # Check exhaustiveness of enum switch statements
@@ -61,9 +56,13 @@ linters:
     - varnamelen # checks that the length of a variable's name matches its scope
     - whitespace # Tool for detection of leading and trailing whitespace
     - wrapcheck # Checks that errors returned from external packages are wrapped
-
+    - staticcheck # Staticcheck is a go vet on steroids, applying a ton of static analysis checks
+
 disable:
   # main-list
+    ## disable now re-enable later
+    ## re-enable never
+    - depguard # Go linter that checks if package imports are in a list of acceptable packages
     - contextcheck # Check the function whether use a non-inherited context
     - cyclop # Checks function and package cyclomatic complexity 
     - durationcheck	# Check for two durations multiplied together

cmd/bspagent/main.go

@@ -1,3 +1,4 @@
+// Decodes, packs, encodes, proves, stores and uploads chain state in avro produced by EVM Execution Layer nodes
 package main
 
 import (

codec/block-ethereum.avsc

@@ -302,6 +302,37 @@
                                        {
                                           "name":"input",
                                           "type":"string"
+                                       },
+                                       {
+                                          "name":"blobFeeCap",
+                                          "type":[
+                                             "null",
+                                             {
+                                                "type":"bytes",
+                                                "logicalType":"decimal",
+                                                "precision":1000
+                                             }
+                                          ],
+                                          "default":"null"
+                                       },
+                                       {
+                                          "name":"blobHashes",
+                                          "type":[
+                                             "null",
+                                             {
+                                                "type":"array",
+                                                "items":"string"
+                                             }
+                                          ],
+                                          "default":"null"
+                                       },
+                                       {
+                                          "name":"blobGas",
+                                          "type":[
+                                             "null",
+                                             "int"
+                                          ],
+                                          "default":"null"
                                        }
                                     ]
                                  }
@@ -544,6 +575,43 @@
                                  }
                               ],
                               "default":"null"
+                           },
+                           {
+                              "name":"BlobTxSidecars",
+                              "type":[
+                                 "null",
+                                 {
+                                    "type":"array",
+                                    "items":{
+                                       "name":"BlobTxSidecars_record",
+                                       "type":"record",
+                                       "fields":[
+                                          {
+                                             "name":"Blobs",
+                                             "type":{
+                                                "type":"array",
+                                                "items":"bytes"
+                                             }
+                                          },
+                                          {
+                                             "name":"Commitments",
+                                             "type":{
+                                                "type":"array",
+                                                "items":"bytes"
+                                             }
+                                          },
+                                          {
+                                             "name":"Proofs",
+                                             "type":{
+                                                "type":"array",
+                                                "items":"bytes"
+                                             }
+                                          }
+                                       ]
+                                    }
+                                 }
+                              ],
+                              "default":"null"
                            }
                         ]
                      }
@@ -567,7 +635,7 @@
       {
          "name":"codecVersion",
          "type":"double",
-         "default":0.36
+         "default":0.37
       }
    ]
 }

codec/codegen.go

@@ -13,34 +13,36 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-/* Code Generation Tool for Go-Avro
+/*Code Generation Tool for Go-Avro
 codegen allows to automatically create Go structs based on defined Avro schema.Usage:
 
 go run codegen.go --schema foo.avsc --schema bar.avsc --out foo.go
 Command line flags:
 --schema - absolute or relative path to Avro schema file. Multiple of those are allowed but at least one is required.
 --out - absolute or relative path to output file. All directories will be created if necessary. Existing file will be truncated.
 */
-
+// code generator for avro schemas
 package main
 
 import (
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/elodina/go-avro"
 )
 
+// schemas is a custom flag type that holds multiple schema file paths.
 type schemas []string
 
+// String returns a string representation of the schemas flag.
 func (i *schemas) String() string {
 	return fmt.Sprintf("%s", *i)
 }
 
+// Set adds a schema file path to the schemas flag.
 func (i *schemas) Set(value string) error {
 	*i = append(*i, value)
 
@@ -55,7 +57,7 @@ func main() {
 
 	var schemas []string
 	for _, schema := range schema {
-		contents, err := ioutil.ReadFile(filepath.Clean(schema))
+		contents, err := os.ReadFile(filepath.Clean(schema))
 		checkErr(err)
 		schemas = append(schemas, string(contents))
 	}
@@ -66,10 +68,11 @@ func main() {
 
 	createDirs()
 	/* #nosec */
-	err = ioutil.WriteFile(*output, []byte(code), 0664)
+	err = os.WriteFile(*output, []byte(code), 0664)
 	checkErr(err)
 }
 
+// parseAndValidateArgs parses and validates the command line arguments.
 func parseAndValidateArgs() {
 	flag.Var(&schema, "schema", "Path to avsc schema file.")
 	flag.Parse()
@@ -85,6 +88,7 @@ func parseAndValidateArgs() {
 	}
 }
 
+// createDirs creates the necessary directories for the output file.
 func createDirs() {
 	index := strings.LastIndex(*output, "/")
 	if index != -1 {
@@ -95,6 +99,7 @@ func createDirs() {
 	}
 }
 
+// checkErr checks if an error occurred and exits the program if it did.
 func checkErr(err error) {
 	if err != nil {
 		fmt.Println(err)

docker-compose-ci.yml

@@ -123,7 +123,7 @@ services:
         sleep 1;
         done;
         echo proof-chain contracts deployed!;
-        ./bsp-agent --redis-url=redis://username:@redis:6379/0?topic=replication-2#replicate --avro-codec-path=./codec/block-ethereum.avsc --binary-file-path=./bin/block-ethereum/  --block-divisor=3  --log-folder ./logs/ --metrics --metrics.port 6063 --metrics.addr 0.0.0.0 --proof-chain-address=0xEa2ff902dbeEECcc828757B881b343F9316752e5 --consumer-timeout=15 --ipfs-pinner-server="http://ipfs-pinner:3001/";
+        ./bsp-agent --redis-url=redis://username:@redis:6379/0?topic=replication#replicate --avro-codec-path=./codec/block-ethereum.avsc --binary-file-path=./bin/block-ethereum/  --block-divisor=3  --log-folder ./logs/ --metrics --metrics.port 6063 --metrics.addr 0.0.0.0 --proof-chain-address=0xEa2ff902dbeEECcc828757B881b343F9316752e5 --consumer-timeout=15 --ipfs-pinner-server="http://ipfs-pinner:3001/";
         exit 0;"
     environment:
       - MB_PRIVATE_KEY=${PRIVATE_KEY}

entry.sh

@@ -9,7 +9,7 @@ then
     --websocket-urls="34.69.250.147:20000 34.69.250.147:20001 34.69.250.147:20002 34.69.250.147:20003"
 
 else
-    ./bsp-agent --redis-url=redis://username:@redis:6379/0?topic=replication-2#replicate \
+    ./bsp-agent --redis-url=redis://username:@redis:6379/0?topic=replication#replicate \
     --avro-codec-path=./codec/block-ethereum.avsc \ 
     --binary-file-path=./bin/block-ethereum/  \
     --block-divisor=3  \

go.mod

@@ -9,9 +9,9 @@ require (
 	github.com/covalenthq/lenspath v0.3.2
 	github.com/covalenthq/lumberjack/v3 v3.0.1
 	github.com/elodina/go-avro v0.0.0-20160406082632-0c8185d9a3ba
-	github.com/ethereum/go-ethereum v1.11.5
+	github.com/ethereum/go-ethereum v1.13.14
 	github.com/go-redis/redis/v7 v7.4.1
-	github.com/golang/snappy v0.0.4
+	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb
 	github.com/gorilla/websocket v1.5.1
 	github.com/ipfs/go-cid v0.1.0
 	github.com/kelseyhightower/envconfig v1.4.0
@@ -26,32 +26,39 @@ require (
 
 require (
 	cloud.google.com/go/bigquery v1.50.0 // indirect
-	github.com/VictoriaMetrics/fastcache v1.10.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 // indirect
 	github.com/apache/arrow/go/v7 v7.0.0 // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
 	github.com/benbjohnson/immutable v0.3.0 // indirect
+	github.com/bits-and-blooms/bitset v1.10.0 // indirect
+	github.com/consensys/bavard v0.1.13 // indirect
+	github.com/consensys/gnark-crypto v0.12.1 // indirect
+	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
-	github.com/edsrzf/mmap-go v1.1.0 // indirect
+	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/gofrs/uuid v3.3.0+incompatible // indirect
-	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/google/flatbuffers v2.0.8+incompatible // indirect
-	github.com/holiman/uint256 v1.2.0 // indirect
+	github.com/holiman/uint256 v1.2.4 // indirect
 	github.com/influxdata/flux v0.170.1 // indirect
 	github.com/influxdata/influxql v1.1.1-0.20211004132434-7e7d61973256 // indirect
 	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
+	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/oapi-codegen/runtime v1.0.0 // indirect
 	github.com/onsi/gomega v1.19.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.15 // indirect
 	github.com/rs/cors v1.8.2 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/supranational/blst v0.3.11 // indirect
 	github.com/uber/jaeger-client-go v2.28.0+incompatible // indirect
 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
 	github.com/xlab/treeprint v1.0.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
 	gonum.org/v1/gonum v0.11.0 // indirect
+	rsc.io/tmplfunc v0.0.3 // indirect
 )
 
 require (
@@ -64,8 +71,7 @@ require (
 	github.com/djherbis/times v1.5.0 // indirect
 	github.com/fatih/color v1.16.0
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
@@ -97,19 +103,18 @@ require (
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/tklauser/go-sysconf v0.3.5 // indirect
-	github.com/tklauser/numcpus v0.2.2 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/ybbus/httpretry v1.0.2
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.21.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/tools v0.15.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
-	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	lukechampine.com/blake3 v1.1.6 // indirect
 )