integrating numscript rewrite

swapping the implementation

update numscript dependency

update numscript version

trigger CI

update numscript

added feature flags and errors

generated stuff

more error handling

run go mod tidy

changed expected error codes

run go mod tidy

revert

fix some tests

run pre-commit

fix err handling

fixes after rebase

cmd/serve.go

@@ -1,16 +1,20 @@
 package cmd
 
 import (
+	"net/http"
+	"net/http/pprof"
+	"time"
+
 	apilib "github.com/formancehq/go-libs/v2/api"
 	"github.com/formancehq/go-libs/v2/health"
 	"github.com/formancehq/go-libs/v2/httpserver"
 	"github.com/formancehq/go-libs/v2/otlp"
+
 	"github.com/formancehq/ledger/internal/storage/driver"
 	"github.com/go-chi/chi/v5"
 	"go.opentelemetry.io/otel/sdk/metric"
-	"net/http"
-	"net/http/pprof"
-	"time"
+
+	"github.com/formancehq/ledger/internal/bus"
 
 	"github.com/formancehq/go-libs/v2/auth"
 	"github.com/formancehq/go-libs/v2/aws/iam"
@@ -19,7 +23,7 @@ import (
 	"github.com/formancehq/go-libs/v2/otlp/otlptraces"
 	"github.com/formancehq/go-libs/v2/publish"
 	"github.com/formancehq/ledger/internal/api"
-	"github.com/formancehq/ledger/internal/bus"
+
 	ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger"
 	systemcontroller "github.com/formancehq/ledger/internal/controller/system"
 	"github.com/formancehq/ledger/internal/storage"
@@ -35,6 +39,7 @@ const (
 	BallastSizeInBytesFlag     = "ballast-size"
 	NumscriptCacheMaxCountFlag = "numscript-cache-max-count"
 	AutoUpgradeFlag            = "auto-upgrade"
+	NumscriptInterpreterFlag   = "numscript-interpreter"
 )
 
 func NewServeCommand() *cobra.Command {
@@ -104,6 +109,7 @@ func NewServeCommand() *cobra.Command {
 	cmd.Flags().Uint(NumscriptCacheMaxCountFlag, 1024, "Numscript cache max count")
 	cmd.Flags().Bool(AutoUpgradeFlag, false, "Automatically upgrade all schemas")
 	cmd.Flags().String(BindFlag, "0.0.0.0:3068", "API bind address")
+	cmd.Flags().Bool(NumscriptInterpreterFlag, false, "Enable experimental numscript rewrite")
 
 	service.AddFlags(cmd.Flags())
 	bunconnect.AddFlags(cmd.Flags())

docs/api/README.md

@@ -3187,6 +3187,8 @@ Authorization ( Scopes: ledger:write )
 |*anonymous*|NO_POSTINGS|
 |*anonymous*|LEDGER_NOT_FOUND|
 |*anonymous*|IMPORT|
+|*anonymous*|INTERPRETER_PARSE|
+|*anonymous*|INTERPRETER_RUNTIME|
 
 <h2 id="tocS_V2LedgerInfoResponse">V2LedgerInfoResponse</h2>
 <!-- backwards compatibility -->

go.mod

@@ -1,6 +1,6 @@
 module github.com/formancehq/ledger
 
-go 1.22.0
+go 1.22.1
 
 toolchain go1.22.7
 
@@ -13,7 +13,9 @@ require (
 	github.com/alitto/pond v1.9.2
 	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10
 	github.com/bluele/gcache v0.0.2
+	github.com/formancehq/go-libs v1.7.2-0.20241014102030-f397c44a5d47
 	github.com/formancehq/go-libs/v2 v2.1.3-0.20241015114549-ea3d600bc087
+	github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417
 	github.com/formancehq/stack/ledger/client v0.0.0-00010101000000-000000000000
 	github.com/go-chi/chi/v5 v5.1.0
 	github.com/go-chi/cors v1.2.1
@@ -56,6 +58,7 @@ require (
 	github.com/ThreeDotsLabs/watermill-kafka/v3 v3.0.5 // indirect
 	github.com/ThreeDotsLabs/watermill-nats/v2 v2.1.1 // indirect
 	github.com/ajg/form v1.5.1 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/aws/aws-msk-iam-sasl-signer-go v1.0.0 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.32.2 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.27.43 // indirect
@@ -179,6 +182,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
 	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/text v0.19.0 // indirect

go.sum

@@ -24,6 +24,8 @@ github.com/alitto/pond v1.9.2 h1:9Qb75z/scEZVCoSU+osVmQ0I0JOeLfdTDafrbcJ8CLs=
 github.com/alitto/pond v1.9.2/go.mod h1:xQn3P/sHTYcU/1BR3i86IGIrilcrGC2LiS+E2+CJWsI=
 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves=
 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
+github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
+github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
 github.com/aws/aws-msk-iam-sasl-signer-go v1.0.0 h1:UyjtGmO0Uwl/K+zpzPwLoXzMhcN9xmnR2nrqJoBrg3c=
 github.com/aws/aws-msk-iam-sasl-signer-go v1.0.0/go.mod h1:TJAXuFs2HcMib3sN5L0gUC+Q01Qvy3DemvA55WuC+iA=
 github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI=
@@ -97,8 +99,18 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/formancehq/go-libs/v2 v2.1.3-0.20241015114549-ea3d600bc087 h1:RT3kp/jbBmKfNCRkLlNiKZk5+BEbkHk4PGU+k1gWFlo=
 github.com/formancehq/go-libs/v2 v2.1.3-0.20241015114549-ea3d600bc087/go.mod h1:NTZO1uTcthk+OKpu+DGoZRkOSV8RVYHZdx0c4wVq5To=
+github.com/formancehq/go-libs v1.7.2-0.20241014102030-f397c44a5d47 h1:FxRRkcLvDj8mrnNQmItxZyrpctJ5KchwTnT9Q1uITrI=
+github.com/formancehq/go-libs v1.7.2-0.20241014102030-f397c44a5d47/go.mod h1:bvfMYwpyRQF6fBkBWty1HiCsKSyYe4RkaEYJEip9Lrw=
+github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417 h1:LOd5hxnXDIBcehFrpW1OnXk+VSs0yJXeu1iAOO+Hji4=
+github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417/go.mod h1:btuSv05cYwi9BvLRxVs5zrunU+O1vTgigG1T6UsawcY=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/gkampitakis/ciinfo v0.3.0 h1:gWZlOC2+RYYttL0hBqcoQhM7h1qNkVqvRCV1fOvpAv8=
+github.com/gkampitakis/ciinfo v0.3.0/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo=
+github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M=
+github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk=
+github.com/gkampitakis/go-snaps v0.5.4 h1:GX+dkKmVsRenz7SoTbdIEL4KQARZctkMiZ8ZKprRwT8=
+github.com/gkampitakis/go-snaps v0.5.4/go.mod h1:ZABkO14uCuVxBHAXAfKG+bqNz+aa1bGPAg8jkI0Nk8Y=
 github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
 github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
 github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
@@ -212,6 +224,8 @@ github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMD
 github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
+github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -295,6 +309,14 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU=
 github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY=
 github.com/tklauser/numcpus v0.9.0 h1:lmyCHtANi8aRUgkckBgoDk1nHCux3n2cgkJLXdQGPDo=
@@ -398,6 +420,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
 golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
+golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=

internal/api/v2/controllers_bulk.go

@@ -108,6 +108,10 @@ func ProcessBulk(
 					code = ErrNoPostings
 				case errors.Is(err, ledgercontroller.ErrTransactionReferenceConflict{}):
 					code = ErrConflict
+				case errors.Is(err, ledgercontroller.ErrParsing{}):
+					code = ErrInterpreterParse
+				case errors.Is(err, ledgercontroller.ErrRuntime{}):
+					code = ErrInterpreterRuntime
 				default:
 					code = api.ErrorInternal
 				}

internal/api/v2/controllers_transactions_create.go

@@ -7,7 +7,9 @@ import (
 	ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger"
 
 	"errors"
+
 	"github.com/formancehq/go-libs/v2/api"
+
 	"github.com/formancehq/ledger/internal/api/common"
 )
 
@@ -50,6 +52,10 @@ func createTransaction(w http.ResponseWriter, r *http.Request) {
 			api.WriteErrorResponse(w, http.StatusConflict, ErrConflict, err)
 		case errors.Is(err, ledgercontroller.ErrInvalidIdempotencyInput{}):
 			api.BadRequest(w, ErrValidation, err)
+		case errors.Is(err, ledgercontroller.ErrParsing{}):
+			api.BadRequest(w, ErrInterpreterParse, err)
+		case errors.Is(err, ledgercontroller.ErrRuntime{}):
+			api.BadRequest(w, ErrInterpreterRuntime, err)
 		default:
 			common.HandleCommonErrors(w, r, err)
 		}

internal/api/v2/errors.go

@@ -8,4 +8,7 @@ const (
 	ErrNoPostings        = "NO_POSTINGS"
 	ErrCompilationFailed = "COMPILATION_FAILED"
 	ErrMetadataOverride  = "METADATA_OVERRIDE"
+
+	ErrInterpreterParse   = "INTERPRETER_PARSE"
+	ErrInterpreterRuntime = "INTERPRETER_RUNTIME"
 )

internal/controller/ledger/errors.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/formancehq/go-libs/v2/platform/postgres"
+	"github.com/formancehq/numscript"
 
 	"github.com/formancehq/ledger/internal/machine"
 
@@ -188,6 +189,36 @@ func newErrCompilationFailed(err error) ErrCompilationFailed {
 	}
 }
 
+type ErrRuntime struct {
+	Source string
+	Inner  numscript.InterpreterError
+}
+
+func (e ErrRuntime) Error() string {
+	return e.Inner.Error()
+}
+
+func (e ErrRuntime) Is(err error) bool {
+	_, ok := err.(ErrRuntime)
+	return ok
+}
+
+type ErrParsing struct {
+	Source string
+	// Precondition: Errors is not empty
+	Errors []numscript.ParserError
+}
+
+func (e ErrParsing) Error() string {
+	// TODO write a decent description
+	return "Got parsing errors"
+}
+
+func (e ErrParsing) Is(err error) bool {
+	_, ok := err.(ErrParsing)
+	return ok
+}
+
 // ErrMetadataOverride is used when a metadata is defined at numscript level AND at the input level
 type ErrMetadataOverride struct {
 	key string

internal/controller/ledger/numscript_runtime.go

@@ -4,15 +4,15 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/formancehq/ledger/internal/machine"
-
 	"errors"
 
 	"github.com/formancehq/go-libs/v2/collectionutils"
 	"github.com/formancehq/go-libs/v2/metadata"
 	ledger "github.com/formancehq/ledger/internal"
+	"github.com/formancehq/ledger/internal/machine"
 	"github.com/formancehq/ledger/internal/machine/vm"
 	"github.com/formancehq/ledger/internal/machine/vm/program"
+	"github.com/formancehq/numscript"
 )
 
 type NumscriptExecutionResult struct {
@@ -86,3 +86,51 @@ func NewMachineNumscriptRuntimeAdapter(p program.Program) *MachineNumscriptRunti
 }
 
 var _ NumscriptRuntime = (*MachineNumscriptRuntimeAdapter)(nil)
+
+// numscript rewrite implementation
+var _ Machine = (*DefaultInterpreterMachineAdapter)(nil)
+
+type DefaultInterpreterMachineAdapter struct {
+	parseResult numscript.ParseResult
+}
+
+func NewDefaultInterpreterMachineAdapter(parseResult numscript.ParseResult) *DefaultInterpreterMachineAdapter {
+	return &DefaultInterpreterMachineAdapter{
+		parseResult: parseResult,
+	}
+}
+
+func (d *DefaultInterpreterMachineAdapter) Execute(ctx context.Context, tx TX, vars map[string]string) (*MachineResult, error) {
+	execResult, err := d.parseResult.Run(ctx, vars, newNumscriptRewriteAdapter(tx))
+	if err != nil {
+		return nil, ErrRuntime{
+			Source: d.parseResult.GetSource(),
+			Inner:  err,
+		}
+	}
+
+	return &MachineResult{
+		Postings: collectionutils.Map(execResult.Postings, func(posting numscript.Posting) ledger.Posting {
+			return ledger.Posting(posting)
+		}),
+		Metadata:        castMetadata(execResult.Metadata),
+		AccountMetadata: castAccountsMetadata(execResult.AccountsMetadata),
+	}, nil
+}
+
+func castMetadata(numscriptMeta numscript.Metadata) metadata.Metadata {
+	meta := metadata.Metadata{}
+	for k, v := range numscriptMeta {
+		meta[k] = v.String()
+	}
+	return meta
+}
+
+func castAccountsMetadata(numscriptAccountsMetadata numscript.AccountsMetadata) map[string]metadata.Metadata {
+	m := make(map[string]metadata.Metadata)
+	for k, v := range numscriptAccountsMetadata {
+		m[k] = v
+	}
+	return m
+
+}

internal/controller/ledger/store.go

@@ -7,6 +7,7 @@ import (
 	"math/big"
 
 	"github.com/formancehq/go-libs/v2/migrations"
+	"github.com/formancehq/numscript"
 
 	"github.com/formancehq/go-libs/v2/bun/bunpaginate"
 	"github.com/formancehq/go-libs/v2/metadata"
@@ -297,3 +298,42 @@ func NewListLedgersQuery(pageSize uint64) ListLedgersQuery {
 		PageSize: pageSize,
 	}
 }
+
+// numscript rewrite implementation
+
+var _ numscript.Store = (*numscriptRewriteAdapter)(nil)
+
+func newNumscriptRewriteAdapter(tx TX) *numscriptRewriteAdapter {
+	return &numscriptRewriteAdapter{
+		TX: tx,
+	}
+}
+
+type numscriptRewriteAdapter struct {
+	TX TX
+}
+
+func (s *numscriptRewriteAdapter) GetBalances(ctx context.Context, q numscript.BalanceQuery) (numscript.Balances, error) {
+	vmBalances, err := s.TX.GetBalances(ctx, BalanceQuery(q))
+	if err != nil {
+		return nil, err
+	}
+	return numscript.Balances(vmBalances), nil
+}
+
+func (s *numscriptRewriteAdapter) GetAccountsMetadata(ctx context.Context, q numscript.MetadataQuery) (numscript.AccountsMetadata, error) {
+	m := numscript.AccountsMetadata{}
+
+	// we ignore the needed metadata values and just return all of them
+	for address := range q {
+		v, err := s.TX.GetAccount(ctx, GetAccountQuery{
+			Addr: address,
+		})
+		if err != nil {
+			return nil, err
+		}
+		m[v.Address] = v.Metadata
+	}
+
+	return m, nil
+}

internal/controller/system/controller.go

@@ -50,7 +50,7 @@ func (ctrl *DefaultController) GetLedgerController(ctx context.Context, name str
 		var ledgerController ledgercontroller.Controller = ledgercontroller.NewDefaultController(
 			*l,
 			store,
-			ledgercontroller.NewDefaultNumscriptParser(),
+			&ledgercontroller.DefaultInterpreterMachineFactory{},
 			ledgercontroller.WithMeter(ctrl.meter),
 		)
 

openapi.yaml

@@ -3450,6 +3450,8 @@ components:
         - NO_POSTINGS
         - LEDGER_NOT_FOUND
         - IMPORT
+        - INTERPRETER_PARSE
+        - INTERPRETER_RUNTIME
       example: VALIDATION
     V2LedgerInfoResponse:
       type: object

openapi/v2.yaml

@@ -1722,6 +1722,8 @@ components:
         - NO_POSTINGS
         - LEDGER_NOT_FOUND
         - IMPORT
+        - INTERPRETER_PARSE
+        - INTERPRETER_RUNTIME
       example: VALIDATION
     V2LedgerInfoResponse:
       type: object