Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 9 additions & 18 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,40 +25,31 @@ path = "src/cli/main.rs"
required-features = ["cli"]

[features]
default = ["cli"]
cli = [
"dep:clap",
"dep:anyhow",
"dep:tiktoken-rs",
"dep:comfy-table",
"dep:ratatui",
"dep:crossterm",
"dep:tui-textarea",
"dep:arboard",
"dep:syntect",
"dep:unicode-width",
"dep:chrono",
]
default = ["cli", "cli-stats", "tui", "tui-clipboard", "tui-time", "parallel"]
cli = ["dep:clap", "dep:anyhow"]
cli-stats = ["cli", "dep:tiktoken-rs", "dep:comfy-table"]
tui = ["dep:anyhow", "dep:ratatui", "dep:crossterm", "dep:tui-textarea"]
tui-clipboard = ["tui", "dep:arboard"]
tui-time = ["tui", "dep:chrono"]
parallel = []

[dependencies]
serde = { version = "1.0.228", features = ["derive"] }
indexmap = "2.0"
serde_json = { version = "1.0.145", features = ["preserve_order"] }
thiserror = "2.0.17"

# CLI dependencies (gated behind "cli" feature)
# CLI dependencies (gated behind "cli"/"cli-stats" features)
clap = { version = "4.5.11", features = ["derive"], optional = true }
anyhow = { version = "1.0.86", optional = true }
tiktoken-rs = { version = "0.9.1", optional = true }
comfy-table = { version = "7.1", optional = true }

# TUI dependencies (gated behind "cli" feature)
# TUI dependencies (gated behind "tui" feature)
ratatui = { version = "0.29", optional = true }
crossterm = { version = "0.28", optional = true }
tui-textarea = { version = "0.7", optional = true }
arboard = { version = "3.4", optional = true }
syntect = { version = "5.2", optional = true }
unicode-width = { version = "0.2", optional = true }
chrono = { version = "0.4", optional = true }

[dev-dependencies]
Expand Down
16 changes: 16 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,22 @@ cargo add toon-format
cargo install toon-format
```

### Feature Flags

By default, all CLI/TUI features are enabled. You can opt in to only what you need:

```toml
toon-format = { version = "0.4", default-features = false }
```

```bash
cargo install toon-format --no-default-features --features cli
cargo install toon-format --no-default-features --features cli,cli-stats
cargo install toon-format --no-default-features --features cli,tui,tui-clipboard,tui-time
```

Feature summary: `cli`, `cli-stats`, `tui`, `tui-clipboard`, `tui-time`, `parallel`.

---

## Library Usage
Expand Down
5 changes: 1 addition & 4 deletions examples/parts/arrays.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
use serde::{
Deserialize,
Serialize,
};
use serde::{Deserialize, Serialize};
use serde_json::json;
use toon_format::encode_default;

Expand Down
5 changes: 1 addition & 4 deletions examples/parts/arrays_of_arrays.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
use serde::{
Deserialize,
Serialize,
};
use serde::{Deserialize, Serialize};
use serde_json::json;
use toon_format::encode_default;

Expand Down
5 changes: 1 addition & 4 deletions examples/parts/decode_strict.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
use serde_json::Value;
use toon_format::{
decode,
DecodeOptions,
};
use toon_format::{decode, DecodeOptions};

pub fn decode_strict() {
// Malformed: header says 2 rows, but only 1 provided
Expand Down
6 changes: 1 addition & 5 deletions examples/parts/delimiters.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
use serde_json::json;
use toon_format::{
encode,
Delimiter,
EncodeOptions,
};
use toon_format::{encode, Delimiter, EncodeOptions};

pub fn delimiters() {
let data = json!({
Expand Down
5 changes: 1 addition & 4 deletions examples/parts/mixed_arrays.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
use serde::{
Deserialize,
Serialize,
};
use serde::{Deserialize, Serialize};
use serde_json::json;
use toon_format::encode_default;

Expand Down
5 changes: 1 addition & 4 deletions examples/parts/objects.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
use serde::{
Deserialize,
Serialize,
};
use serde::{Deserialize, Serialize};
use serde_json::json;
use toon_format::encode_default;

Expand Down
15 changes: 3 additions & 12 deletions examples/parts/round_trip.rs
Original file line number Diff line number Diff line change
@@ -1,15 +1,6 @@
use serde::{
Deserialize,
Serialize,
};
use serde_json::{
json,
Value,
};
use toon_format::{
decode_default,
encode_default,
};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use toon_format::{decode_default, encode_default};

#[derive(Debug, Serialize, Deserialize, PartialEq)]
struct Product {
Expand Down
10 changes: 2 additions & 8 deletions examples/parts/structs.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,5 @@
use serde::{
Deserialize,
Serialize,
};
use toon_format::{
decode_default,
encode_default,
};
use serde::{Deserialize, Serialize};
use toon_format::{decode_default, encode_default};

#[derive(Debug, Serialize, Deserialize, PartialEq)]
struct User {
Expand Down
5 changes: 1 addition & 4 deletions examples/parts/tabular.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
use serde::{
Deserialize,
Serialize,
};
use serde::{Deserialize, Serialize};
use serde_json::json;
use toon_format::encode_default;

Expand Down
113 changes: 57 additions & 56 deletions src/cli/main.rs
Original file line number Diff line number Diff line change
@@ -1,36 +1,19 @@
use std::{
fs,
io::{
self,
Read,
Write,
},
path::{
Path,
PathBuf,
},
io::{self, Read, Write},
path::{Path, PathBuf},
};

use anyhow::{
bail,
Context,
Result,
};
use anyhow::{bail, Context, Result};
use clap::Parser;
#[cfg(feature = "cli-stats")]
use comfy_table::Table;
use serde::Serialize;
#[cfg(feature = "cli-stats")]
use tiktoken_rs::cl100k_base;
use toon_format::{
decode,
encode,
types::{
DecodeOptions,
Delimiter,
EncodeOptions,
Indent,
KeyFoldingMode,
PathExpansionMode,
},
decode, encode,
types::{DecodeOptions, Delimiter, EncodeOptions, Indent, KeyFoldingMode, PathExpansionMode},
};

#[derive(Parser, Debug)]
Expand Down Expand Up @@ -186,39 +169,8 @@ fn run_encode(cli: &Cli, input: &str) -> Result<()> {

write_output(cli.output.clone(), &toon_str)?;

if cli.output.is_none() && !toon_str.ends_with('\n') {
io::stdout().write_all(b"\n")?;
}

if cli.stats {
let json_bytes = input.len();
let toon_bytes = toon_str.len();
let size_savings = 100.0 * (1.0 - (toon_bytes as f64 / json_bytes as f64));

let bpe = cl100k_base().context("Failed to load tokenizer")?;
let json_tokens = bpe.encode_with_special_tokens(input).len();
let toon_tokens = bpe.encode_with_special_tokens(&toon_str).len();
let token_savings = 100.0 * (1.0 - (toon_tokens as f64 / json_tokens as f64));

eprintln!("\nStats:");
let mut table = Table::new();
table.set_header(vec!["Metric", "JSON", "TOON", "Savings"]);

table.add_row(vec![
"Tokens",
&json_tokens.to_string(),
&toon_tokens.to_string(),
&format!("{token_savings:.2}%"),
]);

table.add_row(vec![
"Size (bytes)",
&json_bytes.to_string(),
&toon_bytes.to_string(),
&format!("{size_savings:.2}%"),
]);

eprintln!("\n{table}\n");
render_stats(input, &toon_str)?;
}

Ok(())
Expand Down Expand Up @@ -313,6 +265,11 @@ fn determine_operation(cli: &Cli) -> Result<(Operation, bool)> {
}

fn validate_flags(cli: &Cli, operation: &Operation) -> Result<()> {
#[cfg(not(feature = "cli-stats"))]
if cli.stats {
bail!("--stats requires the 'cli-stats' feature");
}

match operation {
Operation::Encode => {
if cli.no_strict {
Expand Down Expand Up @@ -382,6 +339,50 @@ fn main() -> Result<()> {
Ok(())
}

#[cfg(feature = "cli-stats")]
fn render_stats(input: &str, toon_str: &str) -> Result<()> {
let json_bytes = input.len();
let toon_bytes = toon_str.len();
let size_savings = 100.0 * (1.0 - (toon_bytes as f64 / json_bytes as f64));

let bpe = cl100k_base().context("Failed to load tokenizer")?;
let json_tokens = bpe.encode_with_special_tokens(input).len();
let toon_tokens = bpe.encode_with_special_tokens(toon_str).len();
let token_savings = 100.0 * (1.0 - (toon_tokens as f64 / json_tokens as f64));

eprintln!("\nStats:");
let mut table = Table::new();
table.set_header(vec!["Metric", "JSON", "TOON", "Savings"]);

table.add_row(vec![
"Tokens",
&json_tokens.to_string(),
&toon_tokens.to_string(),
&format!("{token_savings:.2}%"),
]);

table.add_row(vec![
"Size (bytes)",
&json_bytes.to_string(),
&toon_bytes.to_string(),
&format!("{size_savings:.2}%"),
]);

eprintln!("\n{table}\n");
Ok(())
}

#[cfg(not(feature = "cli-stats"))]
fn render_stats(_input: &str, _toon_str: &str) -> Result<()> {
bail!("--stats requires the 'cli-stats' feature");
}

#[cfg(not(feature = "tui"))]
fn run_interactive() -> Result<()> {
bail!("Interactive mode requires the 'tui' feature");
}

#[cfg(feature = "tui")]
fn run_interactive() -> Result<()> {
toon_format::tui::run().context("Failed to run interactive TUI")?;
Ok(())
Expand Down
16 changes: 4 additions & 12 deletions src/decode/expansion.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,7 @@ use indexmap::IndexMap;

use crate::{
constants::QUOTED_KEY_MARKER,
types::{
is_identifier_segment,
JsonValue as Value,
PathExpansionMode,
ToonError,
ToonResult,
},
types::{is_identifier_segment, JsonValue as Value, PathExpansionMode, ToonError, ToonResult},
};

pub fn should_expand_key(key: &str, mode: PathExpansionMode) -> Option<Vec<String>> {
Expand Down Expand Up @@ -90,7 +84,7 @@ pub fn deep_merge_value(
}
} else {
target.insert(first_key.clone(), Value::Object(IndexMap::new()));
match target.get_mut(first_key).unwrap() {
match target.get_mut(first_key).expect("key was just inserted") {
Value::Object(obj) => obj,
_ => unreachable!(),
}
Expand All @@ -108,10 +102,8 @@ pub fn expand_paths_in_object(
let mut result = IndexMap::new();

for (key, mut value) in obj {
// Expand nested objects first (depth-first)
if let Value::Object(nested_obj) = value {
value = Value::Object(expand_paths_in_object(nested_obj, mode, strict)?);
}
// Expand nested structures (arrays/objects) first (depth-first)
value = expand_paths_recursive(value, mode, strict)?;

// Strip marker from quoted keys
let clean_key = if key.starts_with(QUOTED_KEY_MARKER) {
Expand Down
5 changes: 1 addition & 4 deletions src/decode/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,7 @@ pub mod validation;

use serde_json::Value;

use crate::types::{
DecodeOptions,
ToonResult,
};
use crate::types::{DecodeOptions, ToonResult};

/// Decode a TOON string into any deserializable type.
///
Expand Down
Loading
Loading