add first version of rgr sandwich (vercel#3306)

Author: arlyon

.github/workflows/test.yml

@@ -193,6 +193,9 @@ jobs:
       - name: Setup Protos
         run: cd cli && make compile-protos
 
+      - name: Build turborepo-ffi
+        run: cd cli && make turborepo-ffi-install
+
       - name: golangci Linting
         uses: golangci/golangci-lint-action@v3
         with:
@@ -599,6 +602,11 @@ jobs:
       - name: Setup Node.js
         uses: ./.github/actions/setup-node
 
+      - name: Setup Protoc
+        uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Prepare toolchain on Windows
         run: |
           pnpx node-gyp install

.gitignore

@@ -43,3 +43,7 @@ rust-artifacts
 
 # CI
 sweep.timestamp
+
+crates/turborepo-ffi/bindings.h
+crates/turborepo-ffi/ffi/proto/*
+cli/internal/ffi/libturborepo_ffi.a

Cargo.lock

@@ -37,6 +37,7 @@ members = [
   "crates/turbopack",
   "crates/turbopack-tests",
   "crates/turborepo",
+  "crates/turborepo-ffi",
   "crates/turborepo-lib",
   "crates/turbo-updater",
   "xtask",

Cargo.toml

@@ -17,15 +17,33 @@ GO_FLAGS += -trimpath
 
 CLI_DIR = $(shell pwd)
 
+# allow opting in to the rust codepaths
+GO_TAG ?= go
+
 GO_FILES = $(shell find . -name "*.go")
 SRC_FILES = $(shell find . -name "*.go" | grep -v "_test.go")
 GENERATED_FILES = internal/turbodprotocol/turbod.pb.go internal/turbodprotocol/turbod_grpc.pb.go
 
 turbo: go-turbo$(EXT)
 	cargo build --manifest-path ../crates/turborepo/Cargo.toml
 
-go-turbo$(EXT): $(GENERATED_FILES) $(SRC_FILES) go.mod
-	CGO_ENABLED=1 go build $(GO_FLAGS) -o go-turbo$(EXT) ./cmd/turbo
+go-turbo$(EXT): $(GENERATED_FILES) $(SRC_FILES) go.mod turborepo-ffi-install
+	CGO_ENABLED=1 go build $(GO_FLAGS) -tags $(GO_TAG) -o go-turbo$(EXT) ./cmd/turbo
+
+.PHONY: turborepo-ffi-install
+turborepo-ffi-install: turborepo-ffi turborepo-ffi-proto
+	cp -r ../crates/turborepo-ffi/bindings.h \
+	      ../crates/turborepo-ffi/ffi/proto \
+	      ../crates/turborepo-ffi/target/release/libturborepo_ffi.a \
+	      ./internal/ffi
+
+.PHONY: turborepo-ffi
+turborepo-ffi:
+	cd ../crates/turborepo-ffi && CARGO_TARGET_DIR=./target cargo build --release
+
+.PHONY: turborepo-ffi-proto
+turborepo-ffi-proto:
+	cd ../crates/turborepo-ffi && protoc --go_out=. messages.proto
 
 protoc: internal/turbodprotocol/turbod.proto
 	protoc --go_out=. --go_opt=paths=source_relative \
@@ -57,7 +75,7 @@ clean-go:
 	go clean -testcache ./...
 
 test-go: $(GENERATED_FILES) $(GO_FILES) go.mod go.sum
-	go test $(TURBO_RACE) ./...
+	go test $(TURBO_RACE) -tags $(GO_TAG) ./...
 
 # protos need to be compiled before linting, since linting needs to pick up
 # some types from the generated code

cli/Makefile

@@ -0,0 +1,11 @@
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+typedef struct Buffer {
+  uint32_t len;
+  uint8_t *data;
+} Buffer;
+
+struct Buffer get_turbo_data_dir(void);

cli/internal/ffi/bindings.h

@@ -0,0 +1,75 @@
+package ffi
+
+// #include "bindings.h"
+//
+// #cgo LDFLAGS: -L${SRCDIR} -lturborepo_ffi
+// #cgo windows LDFLAGS: -lole32 -lbcrypt -lws2_32 -luserenv
+import "C"
+
+import (
+	"reflect"
+	"unsafe"
+
+	ffi_proto "github.com/vercel/turbo/cli/internal/ffi/proto"
+	"google.golang.org/protobuf/proto"
+)
+
+// Unmarshal consumes a buffer and parses it into a proto.Message
+func Unmarshal[M proto.Message](b C.Buffer, c M) error {
+	bytes := toBytes(b)
+	if err := proto.Unmarshal(bytes, c); err != nil {
+		return err
+	}
+
+	b.Free()
+
+	return nil
+}
+
+// Marshal consumes a proto.Message and returns a bufferfire
+//
+// NOTE: the buffer must be freed by calling `Free` on it
+func Marshal[M proto.Message](c M) C.Buffer {
+	bytes, err := proto.Marshal(c)
+	if err != nil {
+		panic(err)
+	}
+
+	return toBuffer(bytes)
+}
+
+func (c C.Buffer) Free() {
+	C.free(unsafe.Pointer(c.data))
+}
+
+// rather than use C.GoBytes, we use this function to avoid copying the bytes,
+// since it is going to be immediately Unmarshalled into a proto.Message
+func toBytes(b C.Buffer) []byte {
+	var out []byte
+
+	len := (uint32)(b.len)
+
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&out))
+	sh.Data = uintptr(unsafe.Pointer(b.data))
+	sh.Len = int(len)
+	sh.Cap = int(len)
+
+	return out
+}
+
+func toBuffer(bytes []byte) C.Buffer {
+	b := C.Buffer{}
+	b.len = C.uint(len(bytes))
+	b.data = (*C.uchar)(C.CBytes(bytes))
+	return b
+}
+
+// GetTurboDataDir returns the path to the Turbo data directory
+func GetTurboDataDir() string {
+	buffer := C.get_turbo_data_dir()
+	resp := ffi_proto.TurboDataDirResp{}
+	if err := Unmarshal(buffer, resp.ProtoReflect().Interface()); err != nil {
+		panic(err)
+	}
+	return resp.Dir
+}