Skip to content

Commit 7833e98

Browse files
shanithkkshanithkk
committed
fix: large u64 decode issue
1 parent b7cdb5a commit 7833e98

2 files changed

Lines changed: 85 additions & 51 deletions

File tree

‎src/decode/parser.rs‎

Lines changed: 70 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,12 @@
1-
use serde_json::{
2-
Map,
3-
Number,
4-
Value,
5-
};
1+
use serde_json::{Map, Number, Value};
62

73
use crate::{
8-
constants::{
9-
KEYWORDS,
10-
MAX_DEPTH,
11-
QUOTED_KEY_MARKER,
12-
},
4+
constants::{KEYWORDS, MAX_DEPTH, QUOTED_KEY_MARKER},
135
decode::{
14-
scanner::{
15-
Scanner,
16-
Token,
17-
},
6+
scanner::{Scanner, Token},
187
validation,
198
},
20-
types::{
21-
DecodeOptions,
22-
Delimiter,
23-
ErrorContext,
24-
ToonError,
25-
ToonResult,
26-
},
9+
types::{DecodeOptions, Delimiter, ErrorContext, ToonError, ToonResult},
2710
utils::validation::validate_depth,
2811
};
2912

@@ -112,7 +95,7 @@ impl<'a> Parser<'a> {
11295

11396
let had_newline = matches!(self.current_token, Token::Newline);
11497
self.skip_newlines()?;
115-
98+
11699
match &self.current_token {
117100
Token::Null => {
118101
// Peek ahead to see if this is a key (followed by ':') or a value
@@ -142,7 +125,46 @@ impl<'a> Parser<'a> {
142125
Ok(Value::Bool(val))
143126
}
144127
}
145-
Token::Integer(i) => {
128+
Token::SignedInteger(i) => {
129+
let next_char_is_colon = matches!(self.scanner.peek(), Some(':'));
130+
if next_char_is_colon {
131+
let key = i.to_string();
132+
self.advance()?;
133+
self.parse_object_with_initial_key(key, depth)
134+
} else {
135+
let first_text = self.scanner.last_token_text().to_string();
136+
let val = *i;
137+
self.advance()?;
138+
// Check if followed by more value tokens on the same line
139+
match &self.current_token {
140+
Token::String(..)
141+
| Token::SignedInteger(..)
142+
| Token::UnsignedInteger(..)
143+
| Token::Number(..)
144+
| Token::Bool(..)
145+
| Token::Null => {
146+
let mut accumulated = first_text;
147+
while let Token::String(..)
148+
| Token::SignedInteger(..)
149+
| Token::UnsignedInteger(..)
150+
| Token::Number(..)
151+
| Token::Bool(..)
152+
| Token::Null = &self.current_token
153+
{
154+
let ws = self.scanner.last_whitespace_count().max(1);
155+
for _ in 0..ws {
156+
accumulated.push(' ');
157+
}
158+
accumulated.push_str(self.scanner.last_token_text());
159+
self.advance()?;
160+
}
161+
Ok(Value::String(accumulated))
162+
}
163+
_ => Ok(serde_json::Number::from(val).into()),
164+
}
165+
}
166+
}
167+
Token::UnsignedInteger(i) => {
146168
let next_char_is_colon = matches!(self.scanner.peek(), Some(':'));
147169
if next_char_is_colon {
148170
let key = i.to_string();
@@ -155,13 +177,15 @@ impl<'a> Parser<'a> {
155177
// Check if followed by more value tokens on the same line
156178
match &self.current_token {
157179
Token::String(..)
158-
| Token::Integer(..)
180+
| Token::SignedInteger(..)
181+
| Token::UnsignedInteger(..)
159182
| Token::Number(..)
160183
| Token::Bool(..)
161184
| Token::Null => {
162185
let mut accumulated = first_text;
163186
while let Token::String(..)
164-
| Token::Integer(..)
187+
| Token::SignedInteger(..)
188+
| Token::UnsignedInteger(..)
165189
| Token::Number(..)
166190
| Token::Bool(..)
167191
| Token::Null = &self.current_token
@@ -192,13 +216,15 @@ impl<'a> Parser<'a> {
192216
// Check if followed by more value tokens on the same line
193217
match &self.current_token {
194218
Token::String(..)
195-
| Token::Integer(..)
219+
| Token::SignedInteger(..)
220+
| Token::UnsignedInteger(..)
196221
| Token::Number(..)
197222
| Token::Bool(..)
198223
| Token::Null => {
199224
let mut accumulated = first_text;
200225
while let Token::String(..)
201-
| Token::Integer(..)
226+
| Token::SignedInteger(..)
227+
| Token::UnsignedInteger(..)
202228
| Token::Number(..)
203229
| Token::Bool(..)
204230
| Token::Null = &self.current_token
@@ -256,7 +282,8 @@ impl<'a> Parser<'a> {
256282
// Root-level string value - join consecutive tokens with exact spacing
257283
let mut accumulated = first;
258284
while let Token::String(..)
259-
| Token::Integer(..)
285+
| Token::SignedInteger(..)
286+
| Token::UnsignedInteger(..)
260287
| Token::Number(..)
261288
| Token::Bool(..)
262289
| Token::Null = &self.current_token
@@ -502,7 +529,8 @@ impl<'a> Parser<'a> {
502529
// Single token - convert directly to avoid redundant parsing
503530
match &self.current_token {
504531
Token::String(s, _) => Ok(Value::String(s.clone())),
505-
Token::Integer(i) => Ok(serde_json::Number::from(*i).into()),
532+
Token::SignedInteger(i) => Ok(serde_json::Number::from(*i).into()),
533+
Token::UnsignedInteger(i) => Ok(serde_json::Number::from(*i).into()),
506534
Token::Number(n) => {
507535
let val = *n;
508536
if val.is_finite() && val.fract() == 0.0 && val.abs() <= i64::MAX as f64 {
@@ -527,7 +555,8 @@ impl<'a> Parser<'a> {
527555
token_text.clone()
528556
}
529557
Token::String(_, false)
530-
| Token::Integer(_)
558+
| Token::SignedInteger(_)
559+
| Token::UnsignedInteger(_)
531560
| Token::Number(_)
532561
| Token::Bool(_)
533562
| Token::Null => token_text.clone(),
@@ -545,7 +574,8 @@ impl<'a> Parser<'a> {
545574
let token = self.scanner.parse_value_string(&value_str)?;
546575
match token {
547576
Token::String(s, _) => Ok(Value::String(s)),
548-
Token::Integer(i) => Ok(serde_json::Number::from(i).into()),
577+
Token::SignedInteger(i) => Ok(serde_json::Number::from(i).into()),
578+
Token::UnsignedInteger(i) => Ok(serde_json::Number::from(i).into()),
549579
Token::Number(n) => {
550580
if n.is_finite() && n.fract() == 0.0 && n.abs() <= i64::MAX as f64 {
551581
Ok(serde_json::Number::from(n as i64).into())
@@ -588,7 +618,9 @@ impl<'a> Parser<'a> {
588618

589619
// Parse array length (plain integer only)
590620
// Supports formats: [N], [N|], [N\t] (no # marker)
591-
let length = if let Token::Integer(n) = &self.current_token {
621+
let length = if let Token::SignedInteger(n) = &self.current_token {
622+
*n as usize
623+
} else if let Token::UnsignedInteger(n) = &self.current_token {
592624
*n as usize
593625
} else if let Token::String(s, _) = &self.current_token {
594626
// Check if string starts with # - this marker is not supported
@@ -1182,7 +1214,8 @@ impl<'a> Parser<'a> {
11821214
let result = match &self.current_token {
11831215
Token::Null => Ok(Value::Null),
11841216
Token::Bool(b) => Ok(Value::Bool(*b)),
1185-
Token::Integer(i) => Ok(Number::from(*i).into()),
1217+
Token::SignedInteger(i) => Ok(Number::from(*i).into()),
1218+
Token::UnsignedInteger(i) => Ok(Number::from(*i).into()),
11861219
Token::Number(n) => {
11871220
let val = *n;
11881221
if val.is_finite() && val.fract() == 0.0 && val.abs() <= i64::MAX as f64 {
@@ -1216,7 +1249,8 @@ impl<'a> Parser<'a> {
12161249
self.current_token = self.scanner.scan_token()?;
12171250
match token {
12181251
Token::String(s, _) => Ok(Value::String(s)),
1219-
Token::Integer(i) => Ok(Number::from(i).into()),
1252+
Token::SignedInteger(i) => Ok(Number::from(i).into()),
1253+
Token::UnsignedInteger(i) => Ok(Number::from(i).into()),
12201254
Token::Number(n) => {
12211255
if n.is_finite() && n.fract() == 0.0 && n.abs() <= i64::MAX as f64 {
12221256
Ok(Number::from(n as i64).into())
@@ -1244,7 +1278,7 @@ impl<'a> Parser<'a> {
12441278
self.advance()?;
12451279
Ok(Value::Bool(val))
12461280
}
1247-
Token::Integer(i) => {
1281+
Token::SignedInteger(i) => {
12481282
let val = *i;
12491283
self.advance()?;
12501284
Ok(Number::from(val).into())
@@ -1526,10 +1560,7 @@ mod tests {
15261560

15271561
#[test]
15281562
fn test_round_trip_parentheses() {
1529-
use crate::{
1530-
decode::decode_default,
1531-
encode::encode_default,
1532-
};
1563+
use crate::{decode::decode_default, encode::encode_default};
15331564

15341565
let original = json!({
15351566
"message": "Mostly Functions (3 of 3)",

‎src/decode/scanner.rs‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,4 @@
1-
use crate::types::{
2-
Delimiter,
3-
ToonError,
4-
ToonResult,
5-
};
1+
use crate::types::{Delimiter, ToonError, ToonResult};
62

73
/// Tokens produced by the scanner during lexical analysis.
84
#[derive(Debug, Clone, PartialEq)]
@@ -16,14 +12,16 @@ pub enum Token {
1612
Newline,
1713
String(String, bool),
1814
Number(f64),
19-
Integer(i64),
15+
SignedInteger(i64),
16+
UnsignedInteger(u64),
2017
Bool(bool),
2118
Null,
2219
Delimiter(Delimiter),
2320
Eof,
2421
}
2522

2623
/// Scanner that tokenizes TOON input into a sequence of tokens.
24+
#[derive(Debug)]
2725
pub struct Scanner {
2826
input: Vec<char>,
2927
position: usize,
@@ -400,7 +398,9 @@ impl Scanner {
400398
Ok(Token::String(s.to_string(), false))
401399
}
402400
} else if let Ok(i) = s.parse::<i64>() {
403-
Ok(Token::Integer(i))
401+
Ok(Token::SignedInteger(i))
402+
} else if let Ok(i) = s.parse::<u64>() {
403+
Ok(Token::UnsignedInteger(i))
404404
} else {
405405
Ok(Token::String(s.to_string(), false))
406406
}
@@ -547,7 +547,9 @@ impl Scanner {
547547
return Ok(Token::Number(normalized));
548548
}
549549
} else if let Ok(i) = trimmed.parse::<i64>() {
550-
return Ok(Token::Integer(i));
550+
return Ok(Token::SignedInteger(i));
551+
} else if let Ok(i) = trimmed.parse::<u64>() {
552+
return Ok(Token::UnsignedInteger(i));
551553
}
552554
}
553555

@@ -602,13 +604,14 @@ mod tests {
602604

603605
#[test]
604606
fn test_scan_numbers() {
605-
let mut scanner = Scanner::new("42 3.141592653589793 -5");
606-
assert_eq!(scanner.scan_token().unwrap(), Token::Integer(42));
607+
let mut scanner = Scanner::new("42 3.141592653589793 -5 12591154125385152738");
608+
assert_eq!(scanner.scan_token().unwrap(), Token::SignedInteger(42));
607609
assert_eq!(
608610
scanner.scan_token().unwrap(),
609611
Token::Number(f64::consts::PI)
610612
);
611-
assert_eq!(scanner.scan_token().unwrap(), Token::Integer(-5));
613+
assert_eq!(scanner.scan_token().unwrap(), Token::SignedInteger(-5));
614+
assert_eq!(scanner.scan_token().unwrap(), Token::UnsignedInteger(12591154125385152738));
612615
}
613616

614617
#[test]
@@ -717,7 +720,7 @@ mod tests {
717720

718721
assert_eq!(
719722
scanner.parse_value_string("42").unwrap(),
720-
Token::Integer(42)
723+
Token::SignedInteger(42)
721724
);
722725

723726
assert_eq!(

0 commit comments

Comments
 (0)