From 69e9fd7ca759311462ca6c4b5def8bfe748ad2db Mon Sep 17 00:00:00 2001 From: Solype Date: Sun, 8 Nov 2026 13:03:35 +0100 Subject: [PATCH 1/4] refactor(pre-parser): cut a word into a declared name and a particle in cutted_name The search for the longest declared name followed by a name particle moves out of tokenize_names, unchanged. Refs #18 --- src/parser/preparser/pre_parser.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/parser/preparser/pre_parser.rs b/src/parser/preparser/pre_parser.rs index 2e44325..6db060b 100644 --- a/src/parser/preparser/pre_parser.rs +++ b/src/parser/preparser/pre_parser.rs @@ -302,6 +302,17 @@ impl<'a> PreParser<'a> { .any(|token| matches!(&token.kind, LexerTokenKind::HangulWord(word) if word == "반환")) } + fn cutted_name<'w>(word: &'w str, names: &HashSet<&str>) -> Option<(&'w str, Particle)> { + return word + .char_indices() + .rev() + .filter(|(cut, _)| *cut > 0) + .map(|(cut, _)| word.split_at(cut)) + .find_map(|(name, rest)| { + let particle = is_name_particle(rest)?; + names.contains(name).then_some((name, particle)) + }); + } /// Replaces every `Ident` left that is a declared name, alone or followed by a particle a name /// can carry: 나 → Name(나), 나를 → Name(나) + Particle(Object). When several names fit, the /// longest wins (가나를 → 가나 + 를, not 가 + 나를); any other `Ident` stays @@ -321,15 +332,7 @@ impl<'a> PreParser<'a> { continue; } // from the longest name down: 가나를 tries 가나 before 가 - let split = word - .char_indices() - .rev() - .filter(|(cut, _)| *cut > 0) - .map(|(cut, _)| word.split_at(cut)) - .find_map(|(name, rest)| { - let particle = is_name_particle(rest)?; - names.contains(name).then_some((name, particle)) - }); + let split = Self::cutted_name(word, &names); match split { Some((name, particle)) => out.extend(Self::split_name(name, particle, token.span)), None => out.push(token), From 4f929d706d7f9ab9cf0277b068322eee90d9f037 Mon Sep 17 00:00:00 2001 From: Solype Date: Sun, 8 Nov 2026 15:54:03 +0100 Subject: [PATCH 2/4] fix(pre-parser): refuse two names of a scope that differ by a particle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With 사 and 사과 declared in the same function, 사과 can be read as 사 + 과: declared_names now stops with AmbiguousName when a new name is another name of its scope followed by a name particle (가득 next to 가 stays allowed). The scope is the globals plus the current function: it ends at the closing brace, and a new header drops the parameters of one without a body (외부). Fixes #18 --- src/parser/preparser/pre_parse_error.rs | 5 ++ src/parser/preparser/pre_parser.rs | 68 ++++++++++++++++++++++-- src/parser/preparser/pre_parser_tests.rs | 6 ++- 3 files changed, 72 insertions(+), 7 deletions(-) diff --git a/src/parser/preparser/pre_parse_error.rs b/src/parser/preparser/pre_parse_error.rs index 63f9f6b..dc9330f 100644 --- a/src/parser/preparser/pre_parse_error.rs +++ b/src/parser/preparser/pre_parse_error.rs @@ -7,6 +7,8 @@ use crate::lexer::span::Span; pub enum PreParseErrorKind { /// A Hangul word that is no keyword, type, verb or declared name, even cut before a particle UnknownIdentifier(String), + /// A declared name that is another name of the same scope followed by a particle: 사 and 사과 + AmbiguousName(String), } /// A pre-parsing error and where it happened @@ -28,6 +30,9 @@ impl fmt::Display for PreParseErrorKind { Self::UnknownIdentifier(word) => { write!(f, "'{word}' is no keyword, type, verb or declared name") } + Self::AmbiguousName(name) => { + write!(f, "'{name}' can be read as another declared name followed by a particle") + } } } } diff --git a/src/parser/preparser/pre_parser.rs b/src/parser/preparser/pre_parser.rs index 6db060b..463ddab 100644 --- a/src/parser/preparser/pre_parser.rs +++ b/src/parser/preparser/pre_parser.rs @@ -187,7 +187,7 @@ impl<'a> PreParser<'a> { // the last lexer token read: 없는 after a merged 부호 없는 prev = Some(&self.tokens[self.pos - 1].kind); } - let names = Self::declared_names(&tokens); + let names = Self::declared_names(&tokens)?; for (i, replacement) in names.splits.into_iter().rev() { tokens.splice(i..=i, replacement); } @@ -341,15 +341,61 @@ impl<'a> PreParser<'a> { out } + fn is_word_ambiguous(names: &Vec, to_test: &str) -> bool { + for name in names { + if !name.starts_with(to_test) && !to_test.starts_with(name) { + continue; + } + let suffix = if name.len() > to_test.len() { + name.strip_prefix(to_test) + } else { + to_test.strip_prefix(name) + }; + + match suffix { + Some(s) => { + if is_name_particle(s).is_some() { + return true; + } + } + None => {} + } + } + false + } + /// Every declared variable and parameter name, from the pre-parsed tokens (names still `Ident`) - fn declared_names(tokens: &[ParserToken]) -> DeclaredNames { + fn declared_names(tokens: &[ParserToken]) -> Result { let mut names = Vec::new(); let mut splits = BTreeMap::new(); // `두 정수 가와 나를`: after a parameter with 와, the next name may come without its type let mut untyped_next = false; + // the names a new one can conflict with: the globals, then those of the current function + let mut scope: Vec = Vec::new(); + // scope[..globals] are declared outside any function, and stay when one ends + let mut globals = 0; + let mut depth = 0usize; for (i, token) in tokens.iter().enumerate() { - let ParserTokenKind::Ident(word) = &token.kind else { - continue; + let word = match &token.kind { + ParserTokenKind::Ident(word) => word, + ParserTokenKind::Punctuation(Punctuation::LBrace) => { + depth += 1; + continue; + } + // a new header at depth 0: the parameters of a header without a body (외부) go + ParserTokenKind::Punctuation(Punctuation::LParen) if depth == 0 => { + scope.truncate(globals); + continue; + } + ParserTokenKind::Punctuation(Punctuation::RBrace) => { + depth = depth.saturating_sub(1); + // a function ends: its parameters and variables go, the globals stay + if depth == 0 { + scope.truncate(globals); + } + continue; + } + _ => continue, }; let prev = i.checked_sub(1).map(|p| &tokens[p].kind); let next = tokens.get(i + 1).map(|t| &t.kind); @@ -364,6 +410,7 @@ impl<'a> PreParser<'a> { ); if after_a_type && before_comma_or_paren { names.push(word.clone()); + scope.push(word.clone()); continue; } @@ -381,12 +428,23 @@ impl<'a> PreParser<'a> { _ => false, }; if declares && !name.is_empty() { + if Self::is_word_ambiguous(&scope, name) { + return Err(PreParseError::new( + PreParseErrorKind::AmbiguousName(name.to_string()), + token.span, + )); + } names.push(name.to_string()); + scope.push(name.to_string()); + // a variable outside any function is global; a parameter at depth 0 is not + if depth == 0 && particle == Particle::Topic { + globals = scope.len(); + } splits.insert(i, Self::split_name(name, particle, token.span)); untyped_next = particle == Particle::With; } } - DeclaredNames { names, splits } + Ok(DeclaredNames { names, splits }) } /// A declaring word cut after its name, each part with its own span diff --git a/src/parser/preparser/pre_parser_tests.rs b/src/parser/preparser/pre_parser_tests.rs index 9acfc0c..02005ed 100644 --- a/src/parser/preparser/pre_parser_tests.rs +++ b/src/parser/preparser/pre_parser_tests.rs @@ -130,8 +130,10 @@ fn pre_err(src: &str) -> PreParseError { /// The first word of `src` that is no keyword, type, verb or declared name fn unknown(src: &str) -> String { - let PreParseErrorKind::UnknownIdentifier(word) = pre_err(src).kind; - word + match pre_err(src).kind { + PreParseErrorKind::UnknownIdentifier(word) => word, + kind => panic!("{src:?}: {kind:?}, not an unknown identifier"), + } } #[test] From d7da4d240a1445b417a85a1aa6bb5fee800e5eae Mon Sep 17 00:00:00 2001 From: Solype Date: Sun, 8 Nov 2026 16:01:27 +0100 Subject: [PATCH 3/4] fix(pre-parser): check the parameters in parentheses for ambiguous names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 고치다(정수 사, 정수 사과) was accepted: a parameter in parentheses was added to the scope without the check the other declarations get. Unit tests cover AmbiguousName for variables, both kinds of parameters, globals, the scope of a function and a rest that is no name particle (가득, 사로, 사의, 사과일). Refs #18 --- src/parser/preparser/pre_parse_error.rs | 5 ++- src/parser/preparser/pre_parser.rs | 6 +++ src/parser/preparser/pre_parser_tests.rs | 57 ++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/parser/preparser/pre_parse_error.rs b/src/parser/preparser/pre_parse_error.rs index dc9330f..a3ad7f8 100644 --- a/src/parser/preparser/pre_parse_error.rs +++ b/src/parser/preparser/pre_parse_error.rs @@ -31,7 +31,10 @@ impl fmt::Display for PreParseErrorKind { write!(f, "'{word}' is no keyword, type, verb or declared name") } Self::AmbiguousName(name) => { - write!(f, "'{name}' can be read as another declared name followed by a particle") + write!( + f, + "'{name}' can be read as another declared name followed by a particle" + ) } } } diff --git a/src/parser/preparser/pre_parser.rs b/src/parser/preparser/pre_parser.rs index 463ddab..f25e4c2 100644 --- a/src/parser/preparser/pre_parser.rs +++ b/src/parser/preparser/pre_parser.rs @@ -409,6 +409,12 @@ impl<'a> PreParser<'a> { )) ); if after_a_type && before_comma_or_paren { + if Self::is_word_ambiguous(&scope, word) { + return Err(PreParseError::new( + PreParseErrorKind::AmbiguousName(word.to_string()), + token.span, + )); + } names.push(word.clone()); scope.push(word.clone()); continue; diff --git a/src/parser/preparser/pre_parser_tests.rs b/src/parser/preparser/pre_parser_tests.rs index 02005ed..e0d3374 100644 --- a/src/parser/preparser/pre_parser_tests.rs +++ b/src/parser/preparser/pre_parser_tests.rs @@ -533,6 +533,63 @@ fn undeclared_word_is_an_error() { assert_eq!(err.span, Span::new(22, 31)); } +/// The name of `src` that is another name of its scope followed by a particle +fn ambiguous(src: &str) -> String { + match pre_err(src).kind { + PreParseErrorKind::AmbiguousName(name) => name, + kind => panic!("{src:?}: {kind:?}, not an ambiguous name"), + } +} + +#[test] +fn name_that_is_another_name_and_a_particle_is_an_error() { + let err = pre_err("사는 1이에요. 사과는 2예요."); + assert_eq!( + err.kind, + PreParseErrorKind::AmbiguousName("사과".to_string()) + ); + // the whole declaring word: 사과는 at bytes 19..28 + assert_eq!(err.span, Span::new(19, 28)); + // the other way around, 사 is the one that conflicts + assert_eq!(ambiguous("사과는 2예요. 사는 1이에요."), "사"); +} + +#[test] +fn ambiguous_parameters_are_errors() { + assert_eq!(ambiguous("고치다(정수 사, 정수 사과) { }"), "사과"); + assert_eq!(ambiguous("고치다(정수 사) { 사과는 2예요. }"), "사과"); + assert_eq!(ambiguous("정수 사와 정수 사과를 받아 고치고 { }"), "사과"); +} + +#[test] +fn a_global_conflicts_with_the_names_of_a_function() { + assert_eq!(ambiguous("사는 1이에요. main() { 사과는 2예요. }"), "사과"); +} + +#[test] +fn two_functions_are_two_scopes() { + assert_names( + "고치다(정수 사) { } main() { 사과는 2예요. }", + &["사", "사과"], + ); + // a header without a body (외부) keeps its parameters to itself + assert_names( + "외부 정수를 반환하는 abs(정수 사) main() { 사과는 2예요. }", + &["사", "사과"], + ); +} + +#[test] +fn a_rest_that_is_no_name_particle_is_no_conflict() { + // 득 is no particle, 로 and 의 only follow a type, 과일 only starts with one + assert_names("가는 1이에요. 가득은 2예요.", &["가", "가득"]); + assert_names( + "사는 1이에요. 사로는 2예요. 사의는 3이에요. 사과일은 4예요.", + &["사", "사로", "사의", "사과일"], + ); + assert_names("고치다(정수 가, 정수 가득) { }", &["가", "가득"]); +} + #[test] fn no_identifier_left_in_the_prototype() { let tokens = pre(include_str!("../../../Proto.kr")); From 7b04f9d3dca065b5afd43d08ec4e79458f1a07f6 Mon Sep 17 00:00:00 2001 From: Solype Date: Sun, 8 Nov 2026 16:05:02 +0100 Subject: [PATCH 4/4] style(pre-parser): fix the clippy warnings of cutted_name and is_word_ambiguous Refs #18 --- src/parser/preparser/pre_parser.rs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/parser/preparser/pre_parser.rs b/src/parser/preparser/pre_parser.rs index f25e4c2..2c9789b 100644 --- a/src/parser/preparser/pre_parser.rs +++ b/src/parser/preparser/pre_parser.rs @@ -303,15 +303,14 @@ impl<'a> PreParser<'a> { } fn cutted_name<'w>(word: &'w str, names: &HashSet<&str>) -> Option<(&'w str, Particle)> { - return word - .char_indices() + word.char_indices() .rev() .filter(|(cut, _)| *cut > 0) .map(|(cut, _)| word.split_at(cut)) .find_map(|(name, rest)| { let particle = is_name_particle(rest)?; names.contains(name).then_some((name, particle)) - }); + }) } /// Replaces every `Ident` left that is a declared name, alone or followed by a particle a name /// can carry: 나 → Name(나), 나를 → Name(나) + Particle(Object). When several names fit, the @@ -352,13 +351,10 @@ impl<'a> PreParser<'a> { to_test.strip_prefix(name) }; - match suffix { - Some(s) => { - if is_name_particle(s).is_some() { - return true; - } - } - None => {} + if let Some(s) = suffix + && is_name_particle(s).is_some() + { + return true; } } false