expect libtokenizers.a in tokenizer src directory

Author: walterbm-cohere

.gitignore

@@ -9,3 +9,6 @@ target
 
 # Python
 .env
+
+# macos
+.DS_Store

Makefile

@@ -16,10 +16,9 @@ release-darwin-%: test
 	cp artifacts/darwin-$*/libtokenizers.darwin-$*.tar.gz artifacts/all/libtokenizers.darwin-$*.tar.gz
 
 release-linux-%: test
-	docker buildx build --platform linux/$* --build-arg="DOCKER_TARGETPLATFORM=linux/$*" -f release/Dockerfile . -t tokenizers.linux-$*
+	docker buildx build --platform linux/$* -f release/Dockerfile . -t tokenizers.linux-$*
 	mkdir -p artifacts/linux-$*
-	docker run -v $(PWD)/artifacts/linux-$*:/mnt --entrypoint ls tokenizers.linux-$* /workspace/tokenizers/lib/linux
-	docker run -v $(PWD)/artifacts/linux-$*:/mnt --entrypoint cp tokenizers.linux-$* /workspace/tokenizers/lib/linux/$*/libtokenizers.a /mnt/libtokenizers.a
+	docker run -v $(PWD)/artifacts/linux-$*:/mnt --entrypoint cp tokenizers.linux-$* /workspace/tokenizers/libtokenizers.a /mnt/libtokenizers.a
 	cd artifacts/linux-$* && \
 		tar -czf libtokenizers.linux-$*.tar.gz libtokenizers.a
 	mkdir -p artifacts/all
@@ -31,10 +30,10 @@ release: release-darwin-aarch64 release-darwin-x86_64 release-linux-arm64 releas
 	cp artifacts/all/libtokenizers.linux-x86_64.tar.gz artifacts/all/libtokenizers.linux-amd64.tar.gz
 
 test: build
-	@go test -ldflags="-extldflags '-L./'" -v ./... -count=1
+	@go test -v ./... -count=1
 
 clean:
 	rm -rf libtokenizers.a target
 
 bazel-sync:
-	CARGO_BAZEL_REPIN=1 bazel sync --only=crate_index
+	CARGO_BAZEL_REPIN=1 bazel sync --only=crate_index

example/Dockerfile

@@ -8,6 +8,6 @@ RUN curl -fsSL https://github.com/cohere-ai/tokenizers/releases/download/${VERSI
 COPY ./example .
 COPY ./test/data ./test/data
 RUN go mod download
-RUN mv ./libtokenizers.a /go/pkg/mod/github.com/cohere-ai/tokenizers@${VERSION}/lib/$(echo ${TARGETPLATFORM} | tr / -)/libtokenizers.a
+RUN mv ./libtokenizers.a /go/pkg/mod/github.com/daulet/tokenizers@${VERSION}/libtokenizers.a
 # mounting Go cache won't work since we mutate it above
-RUN go run main.go
+RUN go run main.go

release/Dockerfile

@@ -10,7 +10,7 @@ COPY ./Cargo.lock ./Cargo.lock
 RUN cargo build --release
 
 FROM golang:1.21 as builder-go
-ARG DOCKER_TARGETPLATFORM
+ARG TARGETPLATFORM
 WORKDIR /workspace
 COPY ./release/go.mod   .
 COPY ./release/main.go  .
@@ -19,6 +19,6 @@ COPY tokenizer.go       ./tokenizers/
 COPY tokenizers.h       ./tokenizers/
 COPY --from=builder-rust \
     /workspace/target/release/libtokenizers.a \
-    ./tokenizers/lib/${DOCKER_TARGETPLATFORM}/
+    ./tokenizers/
 COPY ./test/data        ./test/data
-RUN go run -ldflags="-extldflags '-L./tokenizers/lib/${DOCKER_TARGETPLATFORM}'" .
+RUN go run .

tokenizer.go

@@ -3,7 +3,7 @@ package tokenizers
 // TODO packaging: how do we build the rust lib for distribution?
 
 /*
-#cgo LDFLAGS: -ltokenizers -ldl -lm -lstdc++
+#cgo LDFLAGS: ${SRCDIR}/libtokenizers.a -ldl -lm -lstdc++
 #include <stdlib.h>
 #include "tokenizers.h"
 */