refactor(turborepo): Two binaries (#3189)

Because of golang/go#13492, we cannot use Go as a shared library that we link to Rust. Instead we are now building the Go code as a separate binary that is executed by the Rust one. Co-authored-by: Greg Soltis <[email protected]> Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> Co-authored-by: Greg Soltis <[email protected]> Co-authored-by: Thomas Knickman <[email protected]>
vercel · Jan 6, 2023 · 251d732 · 251d732
1 parent 03d094f
commit 251d732
Show file tree

Hide file tree

Showing 8 changed files with 86 additions and 188 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/cli/.gitignore b/cli/.gitignore
@@ -9,6 +9,8 @@
 /turbo-new
 /turbo-new.exe
 /turbo.exe
+/go-turbo
+/go-turbo.exe
 
 /scripts/turbo-*
 /.cram_env
@@ -19,4 +21,4 @@ integration_tests/**/*.t.err
 turbo.h
 turbo.lib
 libturbo.a
-libturbo.h
+libturbo.h
diff --git a/cli/Makefile b/cli/Makefile
@@ -1,8 +1,10 @@
 TURBO_VERSION = $(shell cat ../version.txt | sed -n '1 p')
 TURBO_TAG = $(shell cat ../version.txt | sed -n '2 p')
 
+EXT :=
 ifeq ($(OS),Windows_NT)
 	UNAME := Windows
+	EXT = ".exe"
 else
 	UNAME := $(shell uname -s)
 endif
@@ -19,14 +21,11 @@ GO_FILES = $(shell find . -name "*.go")
 SRC_FILES = $(shell find . -name "*.go" | grep -v "_test.go")
 GENERATED_FILES = internal/turbodprotocol/turbod.pb.go internal/turbodprotocol/turbod_grpc.pb.go
 
-turbo: libturbo.a
+turbo: go-turbo$(EXT)
 	cargo build --manifest-path ../crates/turborepo/Cargo.toml
 
-libturbo.a: $(GENERATED_FILES) $(SRC_FILES) go.mod
-	go build -buildmode=c-archive -o libturbo.a  ./cmd/turbo/...
-
-turbo.lib: $(GENERATED_FILES) $(SRC_FILES) go.mod
-	go build -buildmode=c-archive -o turbo.lib  ./cmd/turbo/...
+go-turbo$(EXT): $(GENERATED_FILES) $(SRC_FILES) go.mod
+	CGO_ENABLED=1 go build $(GO_FLAGS) -o go-turbo$(EXT) ./cmd/turbo
 
 protoc: internal/turbodprotocol/turbod.proto
 	protoc --go_out=. --go_opt=paths=source_relative \
@@ -125,6 +124,14 @@ snapshot-lib-turbo-cross:
 build-lib-turbo-darwin:
 	goreleaser release --rm-dist -f darwin-lib.yml
 
+.PHONY: build-go-turbo-darwin
+build-go-turbo-darwin:
+	goreleaser release --rm-dist -f darwin-release.yml
+
+.PHONY: build-go-turbo-cross
+build-go-turbo-cross:
+	goreleaser release --rm-dist -f cross-release.yml
+
 .PHONY: build-lib-turbo-cross
 build-lib-turbo-cross:
 	goreleaser release --rm-dist -f cross-lib.yml
@@ -196,10 +203,10 @@ snapshot-turbo:
 	goreleaser release --rm-dist -f combined-shim.yml --snapshot
 
 	# Split packing from the publish step so that npm locates the correct .npmrc file.
-	npm pack $(CLI_DIR)/../packages/turbo --pack-destination=$(CLI_DIR)/../
-	npm pack $(CLI_DIR)/../packages/turbo-ignore --pack-destination=$(CLI_DIR)/../
-	npm pack $(CLI_DIR)/../packages/create-turbo --pack-destination=$(CLI_DIR)/../
-	npm pack $(CLI_DIR)/../packages/turbo-codemod --pack-destination=$(CLI_DIR)/../
+	npm pack $(CLI_DIR)/../packages/turbo --pack-destination=$(CLI_DIR)/dist/
+	npm pack $(CLI_DIR)/../packages/turbo-ignore --pack-destination=$(CLI_DIR)/dist/
+	npm pack $(CLI_DIR)/../packages/create-turbo --pack-destination=$(CLI_DIR)/dist/
+	npm pack $(CLI_DIR)/../packages/turbo-codemod --pack-destination=$(CLI_DIR)/dist/
 
 
 .PHONY: publish-turbo

diff --git a/cli/cmd/turbo/main.go b/cli/cmd/turbo/main.go
@@ -1,6 +1,5 @@
 package main
 
-import "C"
 import (
 	"encoding/json"
 	"fmt"
@@ -11,18 +10,19 @@ import (
 )
 
 func main() {
-	fmt.Printf("ERROR: Go binary cannot be used on its own. Please build as c-archive and use with Rust crate")
-	os.Exit(1)
-}
+	if len(os.Args) != 2 {
+		fmt.Printf("go-turbo is expected to be invoked via turbo")
+		os.Exit(1)
+	}
 
-//export nativeRunWithArgs
-func nativeRunWithArgs(argsString string) C.uint {
+	argsString := os.Args[1]
 	var args turbostate.ParsedArgsFromRust
 	err := json.Unmarshal([]byte(argsString), &args)
 	if err != nil {
 		fmt.Printf("Error unmarshalling CLI args: %v\n Arg string: %v\n", err, argsString)
-		return 1
+		os.Exit(1)
 	}
+
 	exitCode := cmd.RunWithArgs(args, turboVersion)
-	return C.uint(exitCode)
+	os.Exit(exitCode)
 }
diff --git a/cli/internal/daemon/daemon.go b/cli/internal/daemon/daemon.go
@@ -8,6 +8,8 @@ import (
 	"io"
 	"net"
 	"os"
+	"path/filepath"
+	"strings"
 	"time"
 
 	grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
@@ -297,6 +299,12 @@ func GetClient(ctx context.Context, repoRoot turbopath.AbsoluteSystemPath, logge
 	if err != nil {
 		return nil, err
 	}
+	// The Go binary can no longer be called directly, so we need to route back to the rust wrapper
+	if strings.HasSuffix(bin, "go-turbo") {
+		bin = filepath.Join(filepath.Dir(bin), "turbo")
+	} else if strings.HasSuffix(bin, "go-turbo.exe") {
+		bin = filepath.Join(filepath.Dir(bin), "turbo.exe")
+	}
 	c := &connector.Connector{
 		Logger:       logger.Named("TurbodClient"),
 		Bin:          bin,

diff --git a/crates/turborepo/Cargo.toml b/crates/turborepo/Cargo.toml
@@ -6,7 +6,6 @@ license = "MPL-2.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [build-dependencies]
-bindgen = "0.61.0"
 build-target = "0.4.0"
 
 [dev-dependencies]

diff --git a/crates/turborepo/build.rs b/crates/turborepo/build.rs
@@ -1,110 +1,52 @@
-use std::{
-    env,
-    path::{Path, PathBuf},
-    process::Command,
-};
+use std::{env, fs, path::PathBuf, process::Command};
 
 fn main() {
-    let is_ci_release = matches!(env::var("PROFILE"), Ok(profile) if profile == "release")
-        && env::var("RELEASE_TURBO_CLI")
-            .map(|val| val == "true")
-            .unwrap_or(false);
-    let lib_search_path = if is_ci_release {
-        expect_release_lib()
-    } else {
-        build_debug_libturbo()
-    };
-    println!(
-        "cargo:rerun-if-changed={}",
-        lib_search_path.to_string_lossy()
-    );
-    println!(
-        "cargo:rustc-link-search={}",
-        lib_search_path.to_string_lossy()
-    );
-    println!("cargo:rustc-link-lib=turbo");
-
-    let target = build_target::target().unwrap();
-    let bindings = bindgen::Builder::default()
-        .header(lib_search_path.join("libturbo.h").to_string_lossy())
-        // Tell cargo to invalidate the built crate whenever any of the
-        // included header files changed.
-        .parse_callbacks(Box::new(bindgen::CargoCallbacks))
-        .allowlist_function("nativeRunWithArgs")
-        .allowlist_type("GoString")
-        .generate()
-        .expect("Unable to generate bindings");
-
-    let out_dir = env::var("OUT_DIR").unwrap();
-    let out_path = Path::new(&out_dir).join("bindings.rs");
+    println!("cargo:rerun-if-changed=../../cli");
+    let profile = env::var("PROFILE").unwrap();
+    let is_ci_release =
+        &profile == "release" && matches!(env::var("RELEASE_TURBO_CLI"), Ok(v) if v == "true");
 
-    bindings
-        .write_to_file(out_path)
-        .expect("Couldn't write bindings!");
-
-    if target.os == build_target::Os::MacOs {
-        println!("cargo:rustc-link-lib=framework=cocoa");
-        println!("cargo:rustc-link-lib=framework=security");
+    if !is_ci_release {
+        build_local_go_binary(profile);
     }
 }
 
-fn expect_release_lib() -> PathBuf {
-    // We expect all artifacts to be in the cli path
-    let mut dir = cli_path();
-    let target = build_target::target().unwrap();
-    let platform = match target.os {
-        build_target::Os::MacOs => "darwin",
-        build_target::Os::Windows => "windows",
-        build_target::Os::Linux => "linux",
-        _ => panic!("unsupported target {}", target.triple),
-    };
-    let arch = match target.arch {
-        build_target::Arch::AARCH64 => "arm64",
-        build_target::Arch::X86_64 => "amd64_v1",
-        _ => panic!("unsupported target {}", target.triple),
-    };
-    dir.push("libturbo");
-    // format is ${BUILD_ID}_${OS}_${ARCH}. Build id is, for goreleaser reasons,
-    // turbo-${OS}
-    dir.push(format!("turbo-{platform}_{platform}_{arch}"));
-    dir.push("lib");
-    dir
-}
-
-fn build_debug_libturbo() -> PathBuf {
+fn build_local_go_binary(profile: String) -> PathBuf {
     let cli_path = cli_path();
     let target = build_target::target().unwrap();
     let mut cmd = Command::new("make");
     cmd.current_dir(&cli_path);
-    if target.os == build_target::Os::Windows {
-        let output_dir = env::var_os("OUT_DIR").map(PathBuf::from).unwrap();
-        let output_deps = output_dir
-            .parent()
-            .unwrap()
-            .parent()
-            .unwrap()
-            .parent()
-            .unwrap()
-            .join("deps");
-        // workaround to make increment build works
-        for ext in ["pdb", "exe", "d", "lib"].iter() {
-            std::fs::remove_file(output_deps.join(format!("turbo.{ext}"))).unwrap_or(());
-        }
 
-        cmd.env("CGO_ENABLED", "1")
-            .env("CC", "gcc")
-            .env("CXX", "g++")
-            .arg("turbo.lib");
+    let go_binary_name = if target.os == build_target::Os::Windows {
+        "go-turbo.exe"
     } else {
-        cmd.arg("libturbo.a");
-    }
+        "go-turbo"
+    };
+
+    cmd.arg(go_binary_name);
+
     assert!(
         cmd.stdout(std::process::Stdio::inherit())
             .status()
-            .expect("failed to build turbo.lib")
+            .expect("failed to build go binary")
             .success(),
-        "failed to build turbo static library"
+        "failed to build go binary"
     );
+
+    let go_binary_path = env::var("CARGO_WORKSPACE_DIR")
+        .map(PathBuf::from)
+        .unwrap()
+        .join("cli")
+        .join(go_binary_name);
+
+    let new_go_binary_path = env::var_os("CARGO_WORKSPACE_DIR")
+        .map(PathBuf::from)
+        .unwrap()
+        .join("target")
+        .join(profile)
+        .join(go_binary_name);
+
+    fs::rename(go_binary_path, new_go_binary_path).unwrap();
     cli_path
 }