diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index da837619cd98c..a8c7faa266993 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -193,6 +193,9 @@ jobs: - name: Setup Protos run: cd cli && make compile-protos + - name: Build turborepo-ffi + run: cd cli && make turborepo-ffi-install + - name: golangci Linting uses: golangci/golangci-lint-action@v3 with: @@ -599,6 +602,11 @@ jobs: - name: Setup Node.js uses: ./.github/actions/setup-node + - name: Setup Protoc + uses: arduino/setup-protoc@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Prepare toolchain on Windows run: | pnpx node-gyp install diff --git a/.gitignore b/.gitignore index df0f5b6aeabcb..07a9cd053625e 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ rust-artifacts # CI sweep.timestamp + +crates/turborepo-ffi/bindings.h +crates/turborepo-ffi/ffi/proto/* +cli/internal/ffi/libturborepo_ffi.a diff --git a/Cargo.lock b/Cargo.lock index 7e2b86ef1cf9e..065fc8581b958 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -721,6 +721,25 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2698f953def977c68f935bb0dfa959375ad4638570e969e2f1e9f433cbf1af6" +[[package]] +name = "cbindgen" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6358dedf60f4d9b8db43ad187391afe959746101346fe51bb978126bec61dfb" +dependencies = [ + "clap 3.2.23", + "heck", + "indexmap", + "log", + "proc-macro2 1.0.50", + "quote 1.0.23", + "serde", + "serde_json", + "syn 1.0.107", + "tempfile", + "toml", +] + [[package]] name = "cc" version = "1.0.79" @@ -869,9 +888,12 @@ version = "3.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ + "atty", "bitflags", "clap_lex 0.2.4", "indexmap", + "strsim", + "termcolor", "textwrap 0.16.0", ] @@ -3455,6 +3477,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7843ec2de400bcbc6a6328c958dc38e5359da6e93e72e37bc5246bf1ae776389" +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "napi" version = "2.10.13" @@ -4470,6 +4498,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97e3215779627f01ee256d2fad52f3d95e8e1c11e9fc6fd08f7cd455d5d5c78" +dependencies = [ + "proc-macro2 1.0.50", + "syn 1.0.107", +] + [[package]] name = "priority-queue" version = "1.3.0" @@ -4548,6 +4586,28 @@ dependencies = [ "prost-derive", ] +[[package]] +name = "prost-build" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f8ad728fb08fe212df3c05169e940fbb6d9d16a877ddde14644a983ba2012e" +dependencies = [ + "bytes", + "heck", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 1.0.107", + "tempfile", + "which", +] + [[package]] name = "prost-derive" version = "0.11.6" @@ -7361,6 +7421,7 @@ dependencies = [ "serde_json", "serde_yaml 0.8.26", "tiny-gradient", + "turborepo-ffi", "turborepo-lib", ] @@ -7840,6 +7901,16 @@ dependencies = [ "turbopack-env", ] +[[package]] +name = "turborepo-ffi" +version = "0.1.0" +dependencies = [ + "cbindgen", + "directories", + "prost", + "prost-build", +] + [[package]] name = "turborepo-lib" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 79565361dba2d..664ad5582ae42 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ members = [ "crates/turbopack", "crates/turbopack-tests", "crates/turborepo", + "crates/turborepo-ffi", "crates/turborepo-lib", "crates/turbo-updater", "xtask", diff --git a/cli/Makefile b/cli/Makefile index c32f294692aff..7dce7cb3a455c 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -17,6 +17,9 @@ GO_FLAGS += -trimpath CLI_DIR = $(shell pwd) +# allow opting in to the rust codepaths +GO_TAG ?= go + GO_FILES = $(shell find . -name "*.go") SRC_FILES = $(shell find . -name "*.go" | grep -v "_test.go") GENERATED_FILES = internal/turbodprotocol/turbod.pb.go internal/turbodprotocol/turbod_grpc.pb.go @@ -24,8 +27,23 @@ GENERATED_FILES = internal/turbodprotocol/turbod.pb.go internal/turbodprotocol/t turbo: go-turbo$(EXT) cargo build --manifest-path ../crates/turborepo/Cargo.toml -go-turbo$(EXT): $(GENERATED_FILES) $(SRC_FILES) go.mod - CGO_ENABLED=1 go build $(GO_FLAGS) -o go-turbo$(EXT) ./cmd/turbo +go-turbo$(EXT): $(GENERATED_FILES) $(SRC_FILES) go.mod turborepo-ffi-install + CGO_ENABLED=1 go build $(GO_FLAGS) -tags $(GO_TAG) -o go-turbo$(EXT) ./cmd/turbo + +.PHONY: turborepo-ffi-install +turborepo-ffi-install: turborepo-ffi turborepo-ffi-proto + cp -r ../crates/turborepo-ffi/bindings.h \ + ../crates/turborepo-ffi/ffi/proto \ + ../crates/turborepo-ffi/target/release/libturborepo_ffi.a \ + ./internal/ffi + +.PHONY: turborepo-ffi +turborepo-ffi: + cd ../crates/turborepo-ffi && CARGO_TARGET_DIR=./target cargo build --release + +.PHONY: turborepo-ffi-proto +turborepo-ffi-proto: + cd ../crates/turborepo-ffi && protoc --go_out=. messages.proto protoc: internal/turbodprotocol/turbod.proto protoc --go_out=. --go_opt=paths=source_relative \ @@ -57,7 +75,7 @@ clean-go: go clean -testcache ./... test-go: $(GENERATED_FILES) $(GO_FILES) go.mod go.sum - go test $(TURBO_RACE) ./... + go test $(TURBO_RACE) -tags $(GO_TAG) ./... # protos need to be compiled before linting, since linting needs to pick up # some types from the generated code diff --git a/cli/internal/ffi/bindings.h b/cli/internal/ffi/bindings.h new file mode 100644 index 0000000000000..bffa9cd6849e3 --- /dev/null +++ b/cli/internal/ffi/bindings.h @@ -0,0 +1,11 @@ +#include +#include +#include +#include + +typedef struct Buffer { + uint32_t len; + uint8_t *data; +} Buffer; + +struct Buffer get_turbo_data_dir(void); diff --git a/cli/internal/ffi/ffi.go b/cli/internal/ffi/ffi.go new file mode 100644 index 0000000000000..d4e9d9bfa79ad --- /dev/null +++ b/cli/internal/ffi/ffi.go @@ -0,0 +1,75 @@ +package ffi + +// #include "bindings.h" +// +// #cgo LDFLAGS: -L${SRCDIR} -lturborepo_ffi +// #cgo windows LDFLAGS: -lole32 -lbcrypt -lws2_32 -luserenv +import "C" + +import ( + "reflect" + "unsafe" + + ffi_proto "github.com/vercel/turbo/cli/internal/ffi/proto" + "google.golang.org/protobuf/proto" +) + +// Unmarshal consumes a buffer and parses it into a proto.Message +func Unmarshal[M proto.Message](b C.Buffer, c M) error { + bytes := toBytes(b) + if err := proto.Unmarshal(bytes, c); err != nil { + return err + } + + b.Free() + + return nil +} + +// Marshal consumes a proto.Message and returns a bufferfire +// +// NOTE: the buffer must be freed by calling `Free` on it +func Marshal[M proto.Message](c M) C.Buffer { + bytes, err := proto.Marshal(c) + if err != nil { + panic(err) + } + + return toBuffer(bytes) +} + +func (c C.Buffer) Free() { + C.free(unsafe.Pointer(c.data)) +} + +// rather than use C.GoBytes, we use this function to avoid copying the bytes, +// since it is going to be immediately Unmarshalled into a proto.Message +func toBytes(b C.Buffer) []byte { + var out []byte + + len := (uint32)(b.len) + + sh := (*reflect.SliceHeader)(unsafe.Pointer(&out)) + sh.Data = uintptr(unsafe.Pointer(b.data)) + sh.Len = int(len) + sh.Cap = int(len) + + return out +} + +func toBuffer(bytes []byte) C.Buffer { + b := C.Buffer{} + b.len = C.uint(len(bytes)) + b.data = (*C.uchar)(C.CBytes(bytes)) + return b +} + +// GetTurboDataDir returns the path to the Turbo data directory +func GetTurboDataDir() string { + buffer := C.get_turbo_data_dir() + resp := ffi_proto.TurboDataDirResp{} + if err := Unmarshal(buffer, resp.ProtoReflect().Interface()); err != nil { + panic(err) + } + return resp.Dir +} diff --git a/cli/internal/ffi/proto/messages.pb.go b/cli/internal/ffi/proto/messages.pb.go new file mode 100644 index 0000000000000..011cc1bf56773 --- /dev/null +++ b/cli/internal/ffi/proto/messages.pb.go @@ -0,0 +1,401 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.0 +// protoc v3.20.3 +// source: messages.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type TurboDataDirResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Dir string `protobuf:"bytes,1,opt,name=dir,proto3" json:"dir,omitempty"` +} + +func (x *TurboDataDirResp) Reset() { + *x = TurboDataDirResp{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TurboDataDirResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TurboDataDirResp) ProtoMessage() {} + +func (x *TurboDataDirResp) ProtoReflect() protoreflect.Message { + mi := &file_messages_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TurboDataDirResp.ProtoReflect.Descriptor instead. +func (*TurboDataDirResp) Descriptor() ([]byte, []int) { + return file_messages_proto_rawDescGZIP(), []int{0} +} + +func (x *TurboDataDirResp) GetDir() string { + if x != nil { + return x.Dir + } + return "" +} + +type GlobReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + BasePath string `protobuf:"bytes,1,opt,name=base_path,json=basePath,proto3" json:"base_path,omitempty"` + IncludePatterns []string `protobuf:"bytes,2,rep,name=include_patterns,json=includePatterns,proto3" json:"include_patterns,omitempty"` + ExcludePatterns []string `protobuf:"bytes,3,rep,name=exclude_patterns,json=excludePatterns,proto3" json:"exclude_patterns,omitempty"` + FilesOnly bool `protobuf:"varint,4,opt,name=files_only,json=filesOnly,proto3" json:"files_only,omitempty"` // note that the default for a bool is false +} + +func (x *GlobReq) Reset() { + *x = GlobReq{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GlobReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GlobReq) ProtoMessage() {} + +func (x *GlobReq) ProtoReflect() protoreflect.Message { + mi := &file_messages_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GlobReq.ProtoReflect.Descriptor instead. +func (*GlobReq) Descriptor() ([]byte, []int) { + return file_messages_proto_rawDescGZIP(), []int{1} +} + +func (x *GlobReq) GetBasePath() string { + if x != nil { + return x.BasePath + } + return "" +} + +func (x *GlobReq) GetIncludePatterns() []string { + if x != nil { + return x.IncludePatterns + } + return nil +} + +func (x *GlobReq) GetExcludePatterns() []string { + if x != nil { + return x.ExcludePatterns + } + return nil +} + +func (x *GlobReq) GetFilesOnly() bool { + if x != nil { + return x.FilesOnly + } + return false +} + +type GlobResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Response: + // *GlobResp_Files + // *GlobResp_Error + Response isGlobResp_Response `protobuf_oneof:"response"` +} + +func (x *GlobResp) Reset() { + *x = GlobResp{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GlobResp) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GlobResp) ProtoMessage() {} + +func (x *GlobResp) ProtoReflect() protoreflect.Message { + mi := &file_messages_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GlobResp.ProtoReflect.Descriptor instead. +func (*GlobResp) Descriptor() ([]byte, []int) { + return file_messages_proto_rawDescGZIP(), []int{2} +} + +func (m *GlobResp) GetResponse() isGlobResp_Response { + if m != nil { + return m.Response + } + return nil +} + +func (x *GlobResp) GetFiles() *GlobRespList { + if x, ok := x.GetResponse().(*GlobResp_Files); ok { + return x.Files + } + return nil +} + +func (x *GlobResp) GetError() string { + if x, ok := x.GetResponse().(*GlobResp_Error); ok { + return x.Error + } + return "" +} + +type isGlobResp_Response interface { + isGlobResp_Response() +} + +type GlobResp_Files struct { + Files *GlobRespList `protobuf:"bytes,1,opt,name=files,proto3,oneof"` +} + +type GlobResp_Error struct { + Error string `protobuf:"bytes,2,opt,name=error,proto3,oneof"` +} + +func (*GlobResp_Files) isGlobResp_Response() {} + +func (*GlobResp_Error) isGlobResp_Response() {} + +type GlobRespList struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Files []string `protobuf:"bytes,1,rep,name=files,proto3" json:"files,omitempty"` +} + +func (x *GlobRespList) Reset() { + *x = GlobRespList{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GlobRespList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GlobRespList) ProtoMessage() {} + +func (x *GlobRespList) ProtoReflect() protoreflect.Message { + mi := &file_messages_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GlobRespList.ProtoReflect.Descriptor instead. +func (*GlobRespList) Descriptor() ([]byte, []int) { + return file_messages_proto_rawDescGZIP(), []int{3} +} + +func (x *GlobRespList) GetFiles() []string { + if x != nil { + return x.Files + } + return nil +} + +var File_messages_proto protoreflect.FileDescriptor + +var file_messages_proto_rawDesc = []byte{ + 0x0a, 0x0e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x22, 0x24, 0x0a, 0x10, 0x54, 0x75, 0x72, 0x62, 0x6f, 0x44, 0x61, 0x74, 0x61, 0x44, 0x69, 0x72, + 0x52, 0x65, 0x73, 0x70, 0x12, 0x10, 0x0a, 0x03, 0x64, 0x69, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x64, 0x69, 0x72, 0x22, 0x9b, 0x01, 0x0a, 0x07, 0x47, 0x6c, 0x6f, 0x62, 0x52, + 0x65, 0x71, 0x12, 0x1b, 0x0a, 0x09, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x62, 0x61, 0x73, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, + 0x29, 0x0a, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x74, 0x65, + 0x72, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x78, + 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x61, 0x74, + 0x74, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x5f, 0x6f, + 0x6e, 0x6c, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x73, + 0x4f, 0x6e, 0x6c, 0x79, 0x22, 0x55, 0x0a, 0x08, 0x47, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, + 0x12, 0x25, 0x0a, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0d, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, + 0x52, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x16, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, + 0x0a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x24, 0x0a, 0x0c, 0x47, + 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x66, + 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x66, 0x69, 0x6c, 0x65, + 0x73, 0x42, 0x0b, 0x5a, 0x09, 0x66, 0x66, 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_messages_proto_rawDescOnce sync.Once + file_messages_proto_rawDescData = file_messages_proto_rawDesc +) + +func file_messages_proto_rawDescGZIP() []byte { + file_messages_proto_rawDescOnce.Do(func() { + file_messages_proto_rawDescData = protoimpl.X.CompressGZIP(file_messages_proto_rawDescData) + }) + return file_messages_proto_rawDescData +} + +var file_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_messages_proto_goTypes = []interface{}{ + (*TurboDataDirResp)(nil), // 0: TurboDataDirResp + (*GlobReq)(nil), // 1: GlobReq + (*GlobResp)(nil), // 2: GlobResp + (*GlobRespList)(nil), // 3: GlobRespList +} +var file_messages_proto_depIdxs = []int32{ + 3, // 0: GlobResp.files:type_name -> GlobRespList + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_messages_proto_init() } +func file_messages_proto_init() { + if File_messages_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_messages_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TurboDataDirResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_messages_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GlobReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_messages_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GlobResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_messages_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GlobRespList); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_messages_proto_msgTypes[2].OneofWrappers = []interface{}{ + (*GlobResp_Files)(nil), + (*GlobResp_Error)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_messages_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_messages_proto_goTypes, + DependencyIndexes: file_messages_proto_depIdxs, + MessageInfos: file_messages_proto_msgTypes, + }.Build() + File_messages_proto = out.File + file_messages_proto_rawDesc = nil + file_messages_proto_goTypes = nil + file_messages_proto_depIdxs = nil +} diff --git a/cli/internal/fs/get_turbo_data_dir_go.go b/cli/internal/fs/get_turbo_data_dir_go.go new file mode 100644 index 0000000000000..7bed704f4df9b --- /dev/null +++ b/cli/internal/fs/get_turbo_data_dir_go.go @@ -0,0 +1,16 @@ +//go:build go +// +build go + +package fs + +import ( + "github.com/adrg/xdg" + "github.com/vercel/turbo/cli/internal/turbopath" +) + +// GetTurboDataDir returns a directory outside of the repo +// where turbo can store data files related to turbo. +func GetTurboDataDir() turbopath.AbsoluteSystemPath { + dataHome := AbsoluteSystemPathFromUpstream(xdg.DataHome) + return dataHome.UntypedJoin("turborepo") +} diff --git a/cli/internal/fs/get_turbo_data_dir_rust.go b/cli/internal/fs/get_turbo_data_dir_rust.go new file mode 100644 index 0000000000000..dbc80f3eefad1 --- /dev/null +++ b/cli/internal/fs/get_turbo_data_dir_rust.go @@ -0,0 +1,16 @@ +//go:build rust +// +build rust + +package fs + +import ( + "github.com/vercel/turbo/cli/internal/ffi" + "github.com/vercel/turbo/cli/internal/turbopath" +) + +// GetTurboDataDir returns a directory outside of the repo +// where turbo can store data files related to turbo. +func GetTurboDataDir() turbopath.AbsoluteSystemPath { + dir := ffi.GetTurboDataDir() + return turbopath.AbsoluteSystemPathFromUpstream(dir) +} diff --git a/cli/internal/fs/path.go b/cli/internal/fs/path.go index 9da5e5dc23415..2023d69fd04fe 100644 --- a/cli/internal/fs/path.go +++ b/cli/internal/fs/path.go @@ -105,13 +105,6 @@ func TempDir(subDir string) turbopath.AbsoluteSystemPath { return turbopath.AbsoluteSystemPath(os.TempDir()).UntypedJoin(subDir) } -// GetTurboDataDir returns a directory outside of the repo -// where turbo can store data files related to turbo. -func GetTurboDataDir() turbopath.AbsoluteSystemPath { - dataHome := AbsoluteSystemPathFromUpstream(xdg.DataHome) - return dataHome.UntypedJoin("turborepo") -} - // GetUserConfigDir returns the platform-specific common location // for configuration files that belong to a user. func GetUserConfigDir() turbopath.AbsoluteSystemPath { diff --git a/crates/turborepo-ffi/Cargo.toml b/crates/turborepo-ffi/Cargo.toml new file mode 100644 index 0000000000000..335137a73db28 --- /dev/null +++ b/crates/turborepo-ffi/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "turborepo-ffi" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[lib] +crate-type = ["staticlib"] + +[dependencies] +directories = "4.0.1" +prost = "0.11.6" + +[build-dependencies] +cbindgen = "0.24.3" +prost-build = "0.11.6" diff --git a/crates/turborepo-ffi/build.rs b/crates/turborepo-ffi/build.rs new file mode 100644 index 0000000000000..5cd44b02740dd --- /dev/null +++ b/crates/turborepo-ffi/build.rs @@ -0,0 +1,17 @@ +use std::io::Result; + +use cbindgen::Language; + +fn main() -> Result<()> { + let crate_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + + cbindgen::Builder::new() + .with_crate(crate_dir) + .with_language(Language::C) + .generate() + .expect("Unable to generate bindings") + .write_to_file("bindings.h"); + + prost_build::compile_protos(&["messages.proto"], &["."])?; + Ok(()) +} diff --git a/crates/turborepo-ffi/messages.proto b/crates/turborepo-ffi/messages.proto new file mode 100644 index 0000000000000..4a9759e9256a7 --- /dev/null +++ b/crates/turborepo-ffi/messages.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; +option go_package = "ffi/proto"; + +message TurboDataDirResp { + string dir = 1; +} + +message GlobReq { + string base_path = 1; + repeated string include_patterns = 2; + repeated string exclude_patterns = 3; + bool files_only = 4; // note that the default for a bool is false +} + +message GlobResp { + oneof response { + GlobRespList files = 1; + string error = 2; + } +} + +message GlobRespList { + repeated string files = 1; +} diff --git a/crates/turborepo-ffi/src/lib.rs b/crates/turborepo-ffi/src/lib.rs new file mode 100644 index 0000000000000..84c1e437f1d2d --- /dev/null +++ b/crates/turborepo-ffi/src/lib.rs @@ -0,0 +1,45 @@ +use std::mem::ManuallyDrop; + +mod proto { + include!(concat!(env!("OUT_DIR"), "/_.rs")); +} + +#[repr(C)] +#[derive(Debug)] +pub struct Buffer { + len: u32, + data: *mut u8, +} + +impl From for Buffer { + fn from(value: T) -> Self { + let mut bytes = ManuallyDrop::new(value.encode_to_vec()); + let data = bytes.as_mut_ptr(); + let len = bytes.len() as u32; + Buffer { len, data } + } +} + +impl Buffer { + #[allow(dead_code)] + fn into_proto(self) -> Result { + // SAFETY + // protobuf has a fairly strict schema so overrunning or underrunning the byte + // array will not cause any major issues, that is to say garbage in + // garbage out + let mut slice = unsafe { std::slice::from_raw_parts(self.data, self.len as usize) }; + T::decode(&mut slice) + } +} + +#[no_mangle] +pub extern "C" fn get_turbo_data_dir() -> Buffer { + // note: this is _not_ recommended, but it the current behaviour go-side + // ideally we should use the platform specific convention + // (which we get from using ProjectDirs::from) + let dirs = + directories::ProjectDirs::from_path("turborepo".into()).expect("user has a home dir"); + + let dir = dirs.data_dir().to_string_lossy().to_string(); + proto::TurboDataDirResp { dir }.into() +} diff --git a/crates/turborepo/Cargo.toml b/crates/turborepo/Cargo.toml index 7c7f267dd51ba..d63d31f0a46c9 100644 --- a/crates/turborepo/Cargo.toml +++ b/crates/turborepo/Cargo.toml @@ -12,6 +12,10 @@ rustls-tls = ["turborepo-lib/rustls-tls"] [build-dependencies] build-target = "0.4.0" +# we do not actually depend on this, but need +# to build it before our build.rs script is run +turborepo-ffi = { path = "../turborepo-ffi" } + [dev-dependencies] assert_cmd = "2.0.7" itertools = "0.10.5" diff --git a/scripts/run-example.sh b/scripts/run-example.sh index 6dc7b7eb95858..95a683c2ab619 100755 --- a/scripts/run-example.sh +++ b/scripts/run-example.sh @@ -229,7 +229,7 @@ fi if [[ ! -z $(git status -s) ]];then echo "Detected changes" - git status + git diff exit 1 fi