Initial Rebase

Makefile

@@ -9,7 +9,7 @@ PYTHON?=python3
 build: build-go build-ts
 .PHONY: build
 
-build-go: submodules op-node op-proposer op-batcher
+build-go: op-node op-proposer op-batcher
 .PHONY: build-go
 
 lint-go:

op-alt-da/.gitignore

@@ -0,0 +1 @@
+bin

op-alt-da/Dockerfile

@@ -0,0 +1,9 @@
+ARG OP_STACK_GO_BUILDER=us-docker.pkg.dev/oplabs-tools-artifacts/images/op-stack-go:latest
+FROM $OP_STACK_GO_BUILDER AS builder
+# See "make golang-docker" and /ops/docker/op-stack-go
+
+FROM alpine:3.20
+
+COPY --from=builder /usr/local/bin/da-server /usr/local/bin/da-server
+
+CMD ["da-server"]

op-alt-da/Dockerfile.dockerignore

@@ -0,0 +1,6 @@
+*
+
+!/op-service
+!/op-alt-da
+!/go.mod
+!/go.sum

op-alt-da/Makefile

@@ -0,0 +1,3 @@
+DEPRECATED_TARGETS := da-server clean test
+
+include ../justfiles/deprecated.mk

op-alt-da/cli.go

@@ -0,0 +1,119 @@
+package altda
+
+import (
+	"fmt"
+	"net/url"
+	"time"
+
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	EnabledFlagName               = altDAFlags("enabled")
+	DaServerAddressFlagName       = altDAFlags("da-server")
+	VerifyOnReadFlagName          = altDAFlags("verify-on-read")
+	DaServiceFlagName             = altDAFlags("da-service")
+	PutTimeoutFlagName            = altDAFlags("put-timeout")
+	GetTimeoutFlagName            = altDAFlags("get-timeout")
+	MaxConcurrentRequestsFlagName = altDAFlags("max-concurrent-da-requests")
+)
+
+// altDAFlags returns the flag names for altDA
+func altDAFlags(v string) string {
+	return "altda." + v
+}
+
+func altDAEnvs(envprefix, v string) []string {
+	return []string{envprefix + "_ALTDA_" + v}
+}
+
+func CLIFlags(envPrefix string, category string) []cli.Flag {
+	return []cli.Flag{
+		&cli.BoolFlag{
+			Name:     EnabledFlagName,
+			Usage:    "Enable Alt-DA mode\nAlt-DA Mode is a Beta feature of the MIT licensed OP Stack.  While it has received initial review from core contributors, it is still undergoing testing, and may have bugs or other issues.",
+			Value:    false,
+			EnvVars:  altDAEnvs(envPrefix, "ENABLED"),
+			Category: category,
+		},
+		&cli.StringFlag{
+			Name:     DaServerAddressFlagName,
+			Usage:    "HTTP address of a DA Server",
+			EnvVars:  altDAEnvs(envPrefix, "DA_SERVER"),
+			Category: category,
+		},
+		&cli.BoolFlag{
+			Name:     VerifyOnReadFlagName,
+			Usage:    "Verify input data matches the commitments from the DA storage service",
+			Value:    true,
+			EnvVars:  altDAEnvs(envPrefix, "VERIFY_ON_READ"),
+			Category: category,
+		},
+		&cli.BoolFlag{
+			Name:     DaServiceFlagName,
+			Usage:    "Use DA service type where commitments are generated by Alt-DA server",
+			Value:    false,
+			EnvVars:  altDAEnvs(envPrefix, "DA_SERVICE"),
+			Category: category,
+		},
+		&cli.DurationFlag{
+			Name:     PutTimeoutFlagName,
+			Usage:    "Timeout for put requests. 0 means no timeout.",
+			Value:    time.Duration(0),
+			EnvVars:  altDAEnvs(envPrefix, "PUT_TIMEOUT"),
+			Category: category,
+		},
+		&cli.DurationFlag{
+			Name:     GetTimeoutFlagName,
+			Usage:    "Timeout for get requests. 0 means no timeout.",
+			Value:    time.Duration(0),
+			EnvVars:  altDAEnvs(envPrefix, "GET_TIMEOUT"),
+			Category: category,
+		},
+		&cli.Uint64Flag{
+			Name:     MaxConcurrentRequestsFlagName,
+			Usage:    "Maximum number of concurrent requests to the DA server",
+			Value:    1,
+			EnvVars:  altDAEnvs(envPrefix, "MAX_CONCURRENT_DA_REQUESTS"),
+			Category: category,
+		},
+	}
+}
+
+type CLIConfig struct {
+	Enabled               bool
+	DAServerURL           string
+	VerifyOnRead          bool
+	GenericDA             bool
+	PutTimeout            time.Duration
+	GetTimeout            time.Duration
+	MaxConcurrentRequests uint64
+}
+
+func (c CLIConfig) Check() error {
+	if c.Enabled {
+		if c.DAServerURL == "" {
+			return fmt.Errorf("DA server URL is required when altDA is enabled")
+		}
+		if _, err := url.Parse(c.DAServerURL); err != nil {
+			return fmt.Errorf("DA server URL is invalid: %w", err)
+		}
+	}
+	return nil
+}
+
+func (c CLIConfig) NewDAClient() *DAClient {
+	return &DAClient{url: c.DAServerURL, verify: c.VerifyOnRead, precompute: !c.GenericDA, getTimeout: c.GetTimeout, putTimeout: c.PutTimeout}
+}
+
+func ReadCLIConfig(c *cli.Context) CLIConfig {
+	return CLIConfig{
+		Enabled:               c.Bool(EnabledFlagName),
+		DAServerURL:           c.String(DaServerAddressFlagName),
+		VerifyOnRead:          c.Bool(VerifyOnReadFlagName),
+		GenericDA:             c.Bool(DaServiceFlagName),
+		PutTimeout:            c.Duration(PutTimeoutFlagName),
+		GetTimeout:            c.Duration(GetTimeoutFlagName),
+		MaxConcurrentRequests: c.Uint64(MaxConcurrentRequestsFlagName),
+	}
+}

op-alt-da/cmd/daserver/README.md

@@ -0,0 +1,31 @@
+# AltDA Server
+
+## Introduction
+
+This simple DA server implementation supports local storage via file based storage and remote via S3.
+LevelDB is only recommended for usage in local devnets where connecting to S3 is not convenient.
+See the [S3 doc](https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/) for more information
+on how to configure the S3 client.
+
+## S3 Configuration
+
+Depending on your cloud provider a wide array of configurations are available. The S3 client will
+load configurations from the environment, shared credentials and shared config files.
+Sample environment variables are provided below:
+
+```bash
+export AWS_ACCESS_KEY_ID=YOUR_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_ACCESS_KEY
+export AWS_SESSION_TOKEN=YOUR_SESSION_TOKEN
+export AWS_REGION=YOUR_REGION
+```
+
+You can find out more about AWS authentication [here](https://docs.aws.amazon.com/sdkref/latest/guide/creds-config-files.html).
+
+Additionally, these variables can be used with a google cloud S3 endpoint as well, i.e:
+
+```bash
+export AWS_ENDPOINT_URL="https://storage.googleapis.com"
+export AWS_ACCESS_KEY_ID=YOUR_GOOGLE_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY=YOUR_GOOGLE_ACCESS_KEY_SECRET
+```

op-alt-da/cmd/daserver/entrypoint.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+
+	altda "github.com/ethereum-optimism/optimism/op-alt-da"
+	"github.com/ethereum-optimism/optimism/op-service/ctxinterrupt"
+	oplog "github.com/ethereum-optimism/optimism/op-service/log"
+)
+
+func StartDAServer(cliCtx *cli.Context) error {
+	if err := CheckRequired(cliCtx); err != nil {
+		return err
+	}
+
+	cfg := ReadCLIConfig(cliCtx)
+	if err := cfg.Check(); err != nil {
+		return err
+	}
+
+	logCfg := oplog.ReadCLIConfig(cliCtx)
+
+	l := oplog.NewLogger(oplog.AppOut(cliCtx), logCfg)
+	oplog.SetGlobalLogHandler(l.Handler())
+
+	l.Info("Initializing AltDA server...")
+
+	var store altda.KVStore
+
+	if cfg.FileStoreEnabled() {
+		l.Info("Using file storage", "path", cfg.FileStoreDirPath)
+		store = NewFileStore(cfg.FileStoreDirPath)
+	} else if cfg.S3Enabled() {
+		l.Info("Using S3 storage", "bucket", cfg.S3Config().Bucket)
+		s3, err := NewS3Store(cfg.S3Config())
+		if err != nil {
+			return fmt.Errorf("failed to create S3 store: %w", err)
+		}
+		store = s3
+	}
+
+	server := altda.NewDAServer(cliCtx.String(ListenAddrFlagName), cliCtx.Int(PortFlagName), store, l, cfg.UseGenericComm)
+
+	if err := server.Start(); err != nil {
+		return fmt.Errorf("failed to start the DA server")
+	} else {
+		l.Info("Started DA Server")
+	}
+
+	defer func() {
+		if err := server.Stop(); err != nil {
+			l.Error("failed to stop DA server", "err", err)
+		}
+	}()
+
+	return ctxinterrupt.Wait(cliCtx.Context)
+}

op-alt-da/cmd/daserver/file.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"context"
+	"encoding/hex"
+	"os"
+	"path"
+
+	altda "github.com/ethereum-optimism/optimism/op-alt-da"
+)
+
+type FileStore struct {
+	directory string
+}
+
+func NewFileStore(directory string) *FileStore {
+	return &FileStore{
+		directory: directory,
+	}
+}
+
+func (s *FileStore) Get(ctx context.Context, key []byte) ([]byte, error) {
+	data, err := os.ReadFile(s.fileName(key))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, altda.ErrNotFound
+		}
+		return nil, err
+	}
+	return data, nil
+}
+
+func (s *FileStore) Put(ctx context.Context, key []byte, value []byte) error {
+	return os.WriteFile(s.fileName(key), value, 0600)
+}
+
+func (s *FileStore) fileName(key []byte) string {
+	return path.Join(s.directory, hex.EncodeToString(key))
+}