Skip to content

Commit

Permalink
add first version of rgr sandwich (vercel#3306)
Browse files Browse the repository at this point in the history
  • Loading branch information
arlyon authored Feb 6, 2023
1 parent 54df174 commit d08d6aa
Show file tree
Hide file tree
Showing 17 changed files with 731 additions and 11 deletions.
8 changes: 8 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,9 @@ jobs:
- name: Setup Protos
run: cd cli && make compile-protos

- name: Build turborepo-ffi
run: cd cli && make turborepo-ffi-install

- name: golangci Linting
uses: golangci/golangci-lint-action@v3
with:
Expand Down Expand Up @@ -599,6 +602,11 @@ jobs:
- name: Setup Node.js
uses: ./.github/actions/setup-node

- name: Setup Protoc
uses: arduino/setup-protoc@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}

- name: Prepare toolchain on Windows
run: |
pnpx node-gyp install
Expand Down
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,7 @@ rust-artifacts

# CI
sweep.timestamp

crates/turborepo-ffi/bindings.h
crates/turborepo-ffi/ffi/proto/*
cli/internal/ffi/libturborepo_ffi.a
71 changes: 71 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ members = [
"crates/turbopack",
"crates/turbopack-tests",
"crates/turborepo",
"crates/turborepo-ffi",
"crates/turborepo-lib",
"crates/turbo-updater",
"xtask",
Expand Down
24 changes: 21 additions & 3 deletions cli/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,33 @@ GO_FLAGS += -trimpath

CLI_DIR = $(shell pwd)

# allow opting in to the rust codepaths
GO_TAG ?= go

GO_FILES = $(shell find . -name "*.go")
SRC_FILES = $(shell find . -name "*.go" | grep -v "_test.go")
GENERATED_FILES = internal/turbodprotocol/turbod.pb.go internal/turbodprotocol/turbod_grpc.pb.go

turbo: go-turbo$(EXT)
cargo build --manifest-path ../crates/turborepo/Cargo.toml

go-turbo$(EXT): $(GENERATED_FILES) $(SRC_FILES) go.mod
CGO_ENABLED=1 go build $(GO_FLAGS) -o go-turbo$(EXT) ./cmd/turbo
go-turbo$(EXT): $(GENERATED_FILES) $(SRC_FILES) go.mod turborepo-ffi-install
CGO_ENABLED=1 go build $(GO_FLAGS) -tags $(GO_TAG) -o go-turbo$(EXT) ./cmd/turbo

.PHONY: turborepo-ffi-install
turborepo-ffi-install: turborepo-ffi turborepo-ffi-proto
cp -r ../crates/turborepo-ffi/bindings.h \
../crates/turborepo-ffi/ffi/proto \
../crates/turborepo-ffi/target/release/libturborepo_ffi.a \
./internal/ffi

.PHONY: turborepo-ffi
turborepo-ffi:
cd ../crates/turborepo-ffi && CARGO_TARGET_DIR=./target cargo build --release

.PHONY: turborepo-ffi-proto
turborepo-ffi-proto:
cd ../crates/turborepo-ffi && protoc --go_out=. messages.proto

protoc: internal/turbodprotocol/turbod.proto
protoc --go_out=. --go_opt=paths=source_relative \
Expand Down Expand Up @@ -57,7 +75,7 @@ clean-go:
go clean -testcache ./...

test-go: $(GENERATED_FILES) $(GO_FILES) go.mod go.sum
go test $(TURBO_RACE) ./...
go test $(TURBO_RACE) -tags $(GO_TAG) ./...

# protos need to be compiled before linting, since linting needs to pick up
# some types from the generated code
Expand Down
11 changes: 11 additions & 0 deletions cli/internal/ffi/bindings.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>

typedef struct Buffer {
uint32_t len;
uint8_t *data;
} Buffer;

struct Buffer get_turbo_data_dir(void);
75 changes: 75 additions & 0 deletions cli/internal/ffi/ffi.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package ffi

// #include "bindings.h"
//
// #cgo LDFLAGS: -L${SRCDIR} -lturborepo_ffi
// #cgo windows LDFLAGS: -lole32 -lbcrypt -lws2_32 -luserenv
import "C"

import (
"reflect"
"unsafe"

ffi_proto "github.com/vercel/turbo/cli/internal/ffi/proto"
"google.golang.org/protobuf/proto"
)

// Unmarshal consumes a buffer and parses it into a proto.Message
func Unmarshal[M proto.Message](b C.Buffer, c M) error {
bytes := toBytes(b)
if err := proto.Unmarshal(bytes, c); err != nil {
return err
}

b.Free()

return nil
}

// Marshal consumes a proto.Message and returns a bufferfire
//
// NOTE: the buffer must be freed by calling `Free` on it
func Marshal[M proto.Message](c M) C.Buffer {
bytes, err := proto.Marshal(c)
if err != nil {
panic(err)
}

return toBuffer(bytes)
}

func (c C.Buffer) Free() {
C.free(unsafe.Pointer(c.data))
}

// rather than use C.GoBytes, we use this function to avoid copying the bytes,
// since it is going to be immediately Unmarshalled into a proto.Message
func toBytes(b C.Buffer) []byte {
var out []byte

len := (uint32)(b.len)

sh := (*reflect.SliceHeader)(unsafe.Pointer(&out))
sh.Data = uintptr(unsafe.Pointer(b.data))
sh.Len = int(len)
sh.Cap = int(len)

return out
}

func toBuffer(bytes []byte) C.Buffer {
b := C.Buffer{}
b.len = C.uint(len(bytes))
b.data = (*C.uchar)(C.CBytes(bytes))
return b
}

// GetTurboDataDir returns the path to the Turbo data directory
func GetTurboDataDir() string {
buffer := C.get_turbo_data_dir()
resp := ffi_proto.TurboDataDirResp{}
if err := Unmarshal(buffer, resp.ProtoReflect().Interface()); err != nil {
panic(err)
}
return resp.Dir
}
Loading

0 comments on commit d08d6aa

Please sign in to comment.