Cargo.toml

[workspace]
resolver = "3"
members = [
    "crates/stringcheese",
    "crates/stringcheese-core",
    "crates/stringcheese-corpus",
    "crates/stringcheese-compare",
    "crates/stringcheese-unicode",
    "crates/stringcheese-phonetic",
    "crates/stringcheese-bench",
    "crates/stringcheese-cdc",
    "crates/stringcheese-index",
    "crates/stringcheese-align",
    "crates/stringcheese-diff",
    "crates/stringcheese-segment",
    "crates/stringcheese-stats",
    "crates/stringcheese-ident",
    "crates/stringcheese-escape",
    "crates/stringcheese-ngram",
    "crates/stringcheese-minhash",
    "crates/stringcheese-simhash",
    "crates/stringcheese-winnowing",
    "crates/stringcheese-normalize",
    "crates/stringcheese-textsplit",
    "crates/stringcheese-collate",
    "crates/stringcheese-translit",
    "crates/stringcheese-pattern",
    "crates/stringcheese-pattern-regex",
    "crates/stringcheese-manip",
    "crates/stringcheese-lang",
    "crates/stringcheese-lang-gen",
    "crates/stringcheese-lang-test",
    "crates/stringcheese-detect-script",
    "crates/stringcheese-detect-whatlang",
    "crates/stringcheese-detect-lingua",
    "crates/stringcheese-detect",
    "crates/stringcheese-en",
    "crates/stringcheese-de",
    "crates/stringcheese-es",
    "crates/stringcheese-fr",
    "crates/stringcheese-it",
    "crates/stringcheese-pt",
    "crates/stringcheese-ro",
    "crates/stringcheese-nl",
    "crates/stringcheese-no",
    "crates/stringcheese-nn",
    "crates/stringcheese-pl",
    "crates/stringcheese-ja",
    "crates/stringcheese-ko",
    "crates/stringcheese-zh",
    "crates/stringcheese-am",
    "crates/stringcheese-ar",
    "crates/stringcheese-he",
    "crates/stringcheese-fa",
    "crates/stringcheese-hi",
    "crates/stringcheese-mr",
    "crates/stringcheese-bn",
    "crates/stringcheese-pa",
    "crates/stringcheese-id",
    "crates/stringcheese-ta",
    "crates/stringcheese-ml",
    "crates/stringcheese-tr",
    "crates/stringcheese-fi",
    "crates/stringcheese-et",
    "crates/stringcheese-ru",
    "crates/stringcheese-uk",
    "crates/stringcheese-be",
    "crates/stringcheese-bg",
    "crates/stringcheese-mk",
    "crates/stringcheese-sr",
    "crates/stringcheese-cs",
    "crates/stringcheese-da",
    "crates/stringcheese-is",
    "crates/stringcheese-sk",
    "crates/stringcheese-sv",
    "crates/stringcheese-hu",
    "crates/stringcheese-vi",
    "crates/stringcheese-th",
    "crates/stringcheese-el",
    "crates/stringcheese-hy",
    "crates/stringcheese-ka",
    "crates/stringcheese-tokenizer",
    "crates/stringcheese-tokenizer-hf",
    "crates/stringcheese-tokenizer-hf-native",
    "crates/stringcheese-tokenizer-tiktoken",
    "crates/stringcheese-tokenizer-component",
    "crates/stringcheese-scud",
    "crates/stringcheese-icu-case",
    "crates/stringcheese-icu-case-component",
    "crates/stringcheese-icu-collation",
    "crates/stringcheese-icu-collation-component",
    "crates/stringcheese-icu-plural",
    "crates/stringcheese-icu-number",
    "crates/stringcheese-icu-datetime",
    "crates/stringcheese-icu-datetime-component",
    "crates/stringcheese-icu-segment",
    "crates/stringcheese-icu-segment-component",
    "crates/stringcheese-icu-linebreak",
    "crates/stringcheese-icu-linebreak-component",
]
exclude = [
    "fuzz",
    # Parity harness — deliberately kept out of the main workspace so
    # its opt-in `parity-real-vocab` feature (network fetch +
    # multi-MB tiktoken-rs oracle) never leaks into
    # `cargo test --workspace --all-features`. See the crate's
    # README for the rationale and the CI wiring.
    "crates/stringcheese-tokenizer-tiktoken-conformance",
    # Comparative tokenizer bench — same rationale as the parity
    # harness above: its opt-in `parity-real-vocab` feature pulls in
    # `tokenizers-rs` + `tiktoken-rs` as oracles (~2 min of fresh
    # compile between them and a large transitive tree in
    # `Cargo.lock`), and neither belongs on the default CI hot path.
    # Excluded so `cargo test --workspace --all-features --locked`
    # stays lean; the tokenizer-hf-bench CI job runs this crate
    # explicitly. See the crate's README for the rationale and the
    # CI wiring.
    "crates/stringcheese-tokenizer-hf-bench",
    # Real-vocab cl100k WIT component — sibling of
    # `stringcheese-tokenizer-component` that layers OpenAI's real
    # `mergeable_ranks` blob into the same WIT shape. Excluded for
    # the same reason as the conformance crate: its
    # `parity-real-vocab` feature embeds bytes (via a build.rs cache
    # read) that the session-standing constraint in CLAUDE.md forbids
    # committing. Keeping the crate out of the workspace keeps the
    # main CI's `--workspace --all-features` build offline and free
    # of any real-vocab dependency. The dedicated
    # `wasm-tokenizer-component-cl100k` CI job invokes it explicitly
    # after pre-populating the vocab cache.
    "crates/stringcheese-tokenizer-component-cl100k",
    # stringcheese-tokenizer-component-whisper is excluded for the same
    # reason as its cl100k sibling: `parity-real-vocab` bakes real
    # whisper tokenizer.json bytes into the compiled artifact and
    # CLAUDE.md forbids committing those bytes.  The dedicated
    # wasm-tokenizer-component-whisper CI job invokes it explicitly
    # after populating the cache.
    "crates/stringcheese-tokenizer-component-whisper",
]

[workspace.package]
version = "0.1.0"
edition = "2024"
rust-version = "1.88"
license = "MIT OR Apache-2.0"
authors = ["Zachary Whitley <zachary.whitley@tegmentum.ai>"]
repository = "https://github.com/tegmentum/stringcheese"
readme = "README.md"

[workspace.lints.rust]
missing_docs = "warn"
unsafe_op_in_unsafe_fn = "deny"
rust_2018_idioms = { level = "warn", priority = -1 }

[workspace.lints.clippy]
pedantic = { level = "warn", priority = -1 }
module_name_repetitions = "allow"
missing_errors_doc = "allow"
must_use_candidate = "allow"
# `collapsible_if` — allowed workspace-wide as a style choice. Rust
# 1.88 stabilised let-chains and Rust's newer stable clippy tightened
# `collapsible_if` to flag many `if let X { if cond {}}` cases; the
# mechanical rewrite is a let-chain, but there are far more sites in
# tree than are worth churning through, and the nested form remains
# widely used in the ecosystem.
collapsible_if = "allow"
# `undocumented_unsafe_blocks` — every `unsafe {}` block and `unsafe fn`
# body must carry a `// SAFETY:` comment stating the invariant the caller
# is upholding. The prior workspace-wide unsafe audit confirmed 100%
# coverage; bumping the pedantic lint from `warn` to `deny` locks that
# invariant so any new unsafe site without a SAFETY comment fails the
# build (both locally and on CI, where `-D warnings` is set). Generated
# `src/bindings.rs` files carry a per-file `#[allow]` since wit-bindgen
# output does not emit SAFETY comments.
undocumented_unsafe_blocks = "deny"

[workspace.dependencies]
stringcheese-core = { version = "0.1.0", path = "crates/stringcheese-core", default-features = false }
stringcheese-unicode = { version = "0.1.0", path = "crates/stringcheese-unicode", default-features = false }
stringcheese-phonetic = { version = "0.1.0", path = "crates/stringcheese-phonetic", default-features = false }
stringcheese-cdc = { version = "0.1.0", path = "crates/stringcheese-cdc", default-features = false }
stringcheese-index = { version = "0.1.0", path = "crates/stringcheese-index", default-features = false }
stringcheese-corpus = { version = "0.1.0", path = "crates/stringcheese-corpus", default-features = false }
stringcheese-compare = { version = "0.1.0", path = "crates/stringcheese-compare", default-features = false }
stringcheese-align = { version = "0.1.0", path = "crates/stringcheese-align", default-features = false }
stringcheese-diff = { version = "0.1.0", path = "crates/stringcheese-diff", default-features = false }
stringcheese-segment = { version = "0.1.0", path = "crates/stringcheese-segment", default-features = false }
stringcheese-stats = { version = "0.1.0", path = "crates/stringcheese-stats", default-features = false }
stringcheese-ident = { version = "0.1.0", path = "crates/stringcheese-ident", default-features = false }
stringcheese-escape = { version = "0.1.0", path = "crates/stringcheese-escape", default-features = false }
stringcheese-ngram = { version = "0.1.0", path = "crates/stringcheese-ngram", default-features = false }
stringcheese-minhash = { version = "0.1.0", path = "crates/stringcheese-minhash", default-features = false }
stringcheese-simhash = { version = "0.1.0", path = "crates/stringcheese-simhash", default-features = false }
stringcheese-winnowing = { version = "0.1.0", path = "crates/stringcheese-winnowing", default-features = false }
stringcheese-normalize = { version = "0.1.0", path = "crates/stringcheese-normalize", default-features = false }
stringcheese-textsplit = { version = "0.1.0", path = "crates/stringcheese-textsplit", default-features = false }
stringcheese-collate = { version = "0.1.0", path = "crates/stringcheese-collate", default-features = false }
stringcheese-translit = { version = "0.1.0", path = "crates/stringcheese-translit", default-features = false }
stringcheese-pattern = { version = "0.1.0", path = "crates/stringcheese-pattern", default-features = false }
stringcheese-pattern-regex = { version = "0.1.0", path = "crates/stringcheese-pattern-regex" }
stringcheese-manip = { version = "0.1.0", path = "crates/stringcheese-manip", default-features = false }
stringcheese-lang = { version = "0.1.0", path = "crates/stringcheese-lang", default-features = false }
stringcheese-lang-gen = { version = "0.1.0", path = "crates/stringcheese-lang-gen" }
stringcheese-detect-script = { version = "0.1.0", path = "crates/stringcheese-detect-script", default-features = false }
stringcheese-detect-whatlang = { version = "0.1.0", path = "crates/stringcheese-detect-whatlang", default-features = false }
stringcheese-detect-lingua = { version = "0.1.0", path = "crates/stringcheese-detect-lingua", default-features = false }
stringcheese-detect = { version = "0.1.0", path = "crates/stringcheese-detect", default-features = false }
stringcheese-en = { version = "0.1.0", path = "crates/stringcheese-en", default-features = false }
stringcheese-de = { version = "0.1.0", path = "crates/stringcheese-de", default-features = false }
stringcheese-es = { version = "0.1.0", path = "crates/stringcheese-es", default-features = false }
stringcheese-fr = { version = "0.1.0", path = "crates/stringcheese-fr", default-features = false }
stringcheese-it = { version = "0.1.0", path = "crates/stringcheese-it", default-features = false }
stringcheese-pt = { version = "0.1.0", path = "crates/stringcheese-pt", default-features = false }
stringcheese-ro = { version = "0.1.0", path = "crates/stringcheese-ro", default-features = false }
stringcheese-nl = { version = "0.1.0", path = "crates/stringcheese-nl", default-features = false }
stringcheese-no = { version = "0.1.0", path = "crates/stringcheese-no", default-features = false }
stringcheese-nn = { version = "0.1.0", path = "crates/stringcheese-nn", default-features = false }
stringcheese-pl = { version = "0.1.0", path = "crates/stringcheese-pl", default-features = false }
stringcheese-ja = { version = "0.1.0", path = "crates/stringcheese-ja", default-features = false }
stringcheese-ko = { version = "0.1.0", path = "crates/stringcheese-ko", default-features = false }
stringcheese-zh = { version = "0.1.0", path = "crates/stringcheese-zh", default-features = false }
stringcheese-am = { version = "0.1.0", path = "crates/stringcheese-am", default-features = false }
stringcheese-ar = { version = "0.1.0", path = "crates/stringcheese-ar", default-features = false }
stringcheese-he = { version = "0.1.0", path = "crates/stringcheese-he", default-features = false }
stringcheese-fa = { version = "0.1.0", path = "crates/stringcheese-fa", default-features = false }
stringcheese-hi = { version = "0.1.0", path = "crates/stringcheese-hi", default-features = false }
stringcheese-mr = { version = "0.1.0", path = "crates/stringcheese-mr", default-features = false }
stringcheese-bn = { version = "0.1.0", path = "crates/stringcheese-bn", default-features = false }
stringcheese-pa = { version = "0.1.0", path = "crates/stringcheese-pa", default-features = false }
stringcheese-id = { version = "0.1.0", path = "crates/stringcheese-id", default-features = false }
stringcheese-ta = { version = "0.1.0", path = "crates/stringcheese-ta", default-features = false }
stringcheese-ml = { version = "0.1.0", path = "crates/stringcheese-ml", default-features = false }
stringcheese-tr = { version = "0.1.0", path = "crates/stringcheese-tr", default-features = false }
stringcheese-fi = { version = "0.1.0", path = "crates/stringcheese-fi", default-features = false }
stringcheese-et = { version = "0.1.0", path = "crates/stringcheese-et", default-features = false }
stringcheese-ru = { version = "0.1.0", path = "crates/stringcheese-ru", default-features = false }
stringcheese-uk = { version = "0.1.0", path = "crates/stringcheese-uk", default-features = false }
stringcheese-be = { version = "0.1.0", path = "crates/stringcheese-be", default-features = false }
stringcheese-bg = { version = "0.1.0", path = "crates/stringcheese-bg", default-features = false }
stringcheese-mk = { version = "0.1.0", path = "crates/stringcheese-mk", default-features = false }
stringcheese-sr = { version = "0.1.0", path = "crates/stringcheese-sr", default-features = false }
stringcheese-cs = { version = "0.1.0", path = "crates/stringcheese-cs", default-features = false }
stringcheese-da = { version = "0.1.0", path = "crates/stringcheese-da", default-features = false }
stringcheese-is = { version = "0.1.0", path = "crates/stringcheese-is", default-features = false }
stringcheese-sk = { version = "0.1.0", path = "crates/stringcheese-sk", default-features = false }
stringcheese-sv = { version = "0.1.0", path = "crates/stringcheese-sv", default-features = false }
stringcheese-hu = { version = "0.1.0", path = "crates/stringcheese-hu", default-features = false }
stringcheese-vi = { version = "0.1.0", path = "crates/stringcheese-vi", default-features = false }
stringcheese-th = { version = "0.1.0", path = "crates/stringcheese-th", default-features = false }
stringcheese-el = { version = "0.1.0", path = "crates/stringcheese-el", default-features = false }
stringcheese-hy = { version = "0.1.0", path = "crates/stringcheese-hy", default-features = false }
stringcheese-ka = { version = "0.1.0", path = "crates/stringcheese-ka", default-features = false }
stringcheese-tokenizer = { version = "0.1.0", path = "crates/stringcheese-tokenizer", default-features = false }
stringcheese-tokenizer-hf = { version = "0.1.0", path = "crates/stringcheese-tokenizer-hf", default-features = false }
stringcheese-tokenizer-hf-native = { version = "0.1.0", path = "crates/stringcheese-tokenizer-hf-native", default-features = false }
stringcheese-tokenizer-tiktoken = { version = "0.1.0", path = "crates/stringcheese-tokenizer-tiktoken", default-features = false }
stringcheese-tokenizer-component = { version = "0.1.0", path = "crates/stringcheese-tokenizer-component", default-features = false }
stringcheese-scud = { version = "0.1.0", path = "crates/stringcheese-scud", default-features = false }
stringcheese-icu-case = { version = "0.1.0", path = "crates/stringcheese-icu-case", default-features = false }
stringcheese-icu-case-component = { version = "0.1.0", path = "crates/stringcheese-icu-case-component", default-features = false }
stringcheese-icu-collation = { version = "0.1.0", path = "crates/stringcheese-icu-collation", default-features = false }
stringcheese-icu-collation-component = { version = "0.1.0", path = "crates/stringcheese-icu-collation-component", default-features = false }
stringcheese-icu-plural = { version = "0.1.0", path = "crates/stringcheese-icu-plural", default-features = false }
stringcheese-icu-number = { version = "0.1.0", path = "crates/stringcheese-icu-number", default-features = false }
stringcheese-icu-datetime = { version = "0.1.0", path = "crates/stringcheese-icu-datetime", default-features = false }
stringcheese-icu-datetime-component = { version = "0.1.0", path = "crates/stringcheese-icu-datetime-component", default-features = false }
stringcheese-icu-segment = { version = "0.1.0", path = "crates/stringcheese-icu-segment", default-features = false }
stringcheese-icu-segment-component = { version = "0.1.0", path = "crates/stringcheese-icu-segment-component", default-features = false }
stringcheese-icu-linebreak = { version = "0.1.0", path = "crates/stringcheese-icu-linebreak", default-features = false }
stringcheese-icu-linebreak-component = { version = "0.1.0", path = "crates/stringcheese-icu-linebreak-component", default-features = false }

# `criterion` — shared bench harness for the compute-heavy non-tokenizer
# crates. Kept at workspace scope so every crate that adopts a bench
# tracks the same version. Initial adopters: `stringcheese-manip`,
# `stringcheese-compare`, `stringcheese-align`, `stringcheese-cdc`,
# `stringcheese-index`. The workspace-external `stringcheese-tokenizer-hf-bench`
# and `stringcheese-bench` pin the same 0.5 line independently — see
# their manifests for the rationale.
criterion = { version = "0.5", default-features = false, features = ["html_reports"] }

[profile.release]
lto = "thin"
codegen-units = 1