Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/parser/preparser/pre_parse_error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ use crate::lexer::span::Span;
pub enum PreParseErrorKind {
/// A Hangul word that is no keyword, type, verb or declared name, even cut before a particle
UnknownIdentifier(String),
/// A declared name that is another name of the same scope followed by a particle: 사 and 사과
AmbiguousName(String),
}

/// A pre-parsing error and where it happened
Expand All @@ -28,6 +30,12 @@ impl fmt::Display for PreParseErrorKind {
Self::UnknownIdentifier(word) => {
write!(f, "'{word}' is no keyword, type, verb or declared name")
}
Self::AmbiguousName(name) => {
write!(
f,
"'{name}' can be read as another declared name followed by a particle"
)
}
}
}
}
Expand Down
91 changes: 77 additions & 14 deletions src/parser/preparser/pre_parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ impl<'a> PreParser<'a> {
// the last lexer token read: 없는 after a merged 부호 없는
prev = Some(&self.tokens[self.pos - 1].kind);
}
let names = Self::declared_names(&tokens);
let names = Self::declared_names(&tokens)?;
for (i, replacement) in names.splits.into_iter().rev() {
tokens.splice(i..=i, replacement);
}
Expand Down Expand Up @@ -302,6 +302,16 @@ impl<'a> PreParser<'a> {
.any(|token| matches!(&token.kind, LexerTokenKind::HangulWord(word) if word == "반환"))
}

fn cutted_name<'w>(word: &'w str, names: &HashSet<&str>) -> Option<(&'w str, Particle)> {
word.char_indices()
.rev()
.filter(|(cut, _)| *cut > 0)
.map(|(cut, _)| word.split_at(cut))
.find_map(|(name, rest)| {
let particle = is_name_particle(rest)?;
names.contains(name).then_some((name, particle))
})
}
/// Replaces every `Ident` left that is a declared name, alone or followed by a particle a name
/// can carry: 나 → Name(나), 나를 → Name(나) + Particle(Object). When several names fit, the
/// longest wins (가나를 → 가나 + 를, not 가 + 나를); any other `Ident` stays
Expand All @@ -321,15 +331,7 @@ impl<'a> PreParser<'a> {
continue;
}
// from the longest name down: 가나를 tries 가나 before 가
let split = word
.char_indices()
.rev()
.filter(|(cut, _)| *cut > 0)
.map(|(cut, _)| word.split_at(cut))
.find_map(|(name, rest)| {
let particle = is_name_particle(rest)?;
names.contains(name).then_some((name, particle))
});
let split = Self::cutted_name(word, &names);
match split {
Some((name, particle)) => out.extend(Self::split_name(name, particle, token.span)),
None => out.push(token),
Expand All @@ -338,15 +340,58 @@ impl<'a> PreParser<'a> {
out
}

fn is_word_ambiguous(names: &Vec<String>, to_test: &str) -> bool {
for name in names {
if !name.starts_with(to_test) && !to_test.starts_with(name) {
continue;
}
let suffix = if name.len() > to_test.len() {
name.strip_prefix(to_test)
} else {
to_test.strip_prefix(name)
};

if let Some(s) = suffix
&& is_name_particle(s).is_some()
{
return true;
}
}
false
}

/// Every declared variable and parameter name, from the pre-parsed tokens (names still `Ident`)
fn declared_names(tokens: &[ParserToken]) -> DeclaredNames {
fn declared_names(tokens: &[ParserToken]) -> Result<DeclaredNames, PreParseError> {
let mut names = Vec::new();
let mut splits = BTreeMap::new();
// `두 정수 가와 나를`: after a parameter with 와, the next name may come without its type
let mut untyped_next = false;
// the names a new one can conflict with: the globals, then those of the current function
let mut scope: Vec<String> = Vec::new();
// scope[..globals] are declared outside any function, and stay when one ends
let mut globals = 0;
let mut depth = 0usize;
for (i, token) in tokens.iter().enumerate() {
let ParserTokenKind::Ident(word) = &token.kind else {
continue;
let word = match &token.kind {
ParserTokenKind::Ident(word) => word,
ParserTokenKind::Punctuation(Punctuation::LBrace) => {
depth += 1;
continue;
}
// a new header at depth 0: the parameters of a header without a body (외부) go
ParserTokenKind::Punctuation(Punctuation::LParen) if depth == 0 => {
scope.truncate(globals);
continue;
}
ParserTokenKind::Punctuation(Punctuation::RBrace) => {
depth = depth.saturating_sub(1);
// a function ends: its parameters and variables go, the globals stay
if depth == 0 {
scope.truncate(globals);
}
continue;
}
_ => continue,
};
let prev = i.checked_sub(1).map(|p| &tokens[p].kind);
let next = tokens.get(i + 1).map(|t| &t.kind);
Expand All @@ -360,7 +405,14 @@ impl<'a> PreParser<'a> {
))
);
if after_a_type && before_comma_or_paren {
if Self::is_word_ambiguous(&scope, word) {
return Err(PreParseError::new(
PreParseErrorKind::AmbiguousName(word.to_string()),
token.span,
));
}
names.push(word.clone());
scope.push(word.clone());
continue;
}

Expand All @@ -378,12 +430,23 @@ impl<'a> PreParser<'a> {
_ => false,
};
if declares && !name.is_empty() {
if Self::is_word_ambiguous(&scope, name) {
return Err(PreParseError::new(
PreParseErrorKind::AmbiguousName(name.to_string()),
token.span,
));
}
names.push(name.to_string());
scope.push(name.to_string());
// a variable outside any function is global; a parameter at depth 0 is not
if depth == 0 && particle == Particle::Topic {
globals = scope.len();
}
splits.insert(i, Self::split_name(name, particle, token.span));
untyped_next = particle == Particle::With;
}
}
DeclaredNames { names, splits }
Ok(DeclaredNames { names, splits })
}

/// A declaring word cut after its name, each part with its own span
Expand Down
63 changes: 61 additions & 2 deletions src/parser/preparser/pre_parser_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,10 @@ fn pre_err(src: &str) -> PreParseError {

/// The first word of `src` that is no keyword, type, verb or declared name
fn unknown(src: &str) -> String {
let PreParseErrorKind::UnknownIdentifier(word) = pre_err(src).kind;
word
match pre_err(src).kind {
PreParseErrorKind::UnknownIdentifier(word) => word,
kind => panic!("{src:?}: {kind:?}, not an unknown identifier"),
}
}

#[test]
Expand Down Expand Up @@ -531,6 +533,63 @@ fn undeclared_word_is_an_error() {
assert_eq!(err.span, Span::new(22, 31));
}

/// The name of `src` that is another name of its scope followed by a particle
fn ambiguous(src: &str) -> String {
match pre_err(src).kind {
PreParseErrorKind::AmbiguousName(name) => name,
kind => panic!("{src:?}: {kind:?}, not an ambiguous name"),
}
}

#[test]
fn name_that_is_another_name_and_a_particle_is_an_error() {
let err = pre_err("사는 1이에요. 사과는 2예요.");
assert_eq!(
err.kind,
PreParseErrorKind::AmbiguousName("사과".to_string())
);
// the whole declaring word: 사과는 at bytes 19..28
assert_eq!(err.span, Span::new(19, 28));
// the other way around, 사 is the one that conflicts
assert_eq!(ambiguous("사과는 2예요. 사는 1이에요."), "사");
}

#[test]
fn ambiguous_parameters_are_errors() {
assert_eq!(ambiguous("고치다(정수 사, 정수 사과) { }"), "사과");
assert_eq!(ambiguous("고치다(정수 사) { 사과는 2예요. }"), "사과");
assert_eq!(ambiguous("정수 사와 정수 사과를 받아 고치고 { }"), "사과");
}

#[test]
fn a_global_conflicts_with_the_names_of_a_function() {
assert_eq!(ambiguous("사는 1이에요. main() { 사과는 2예요. }"), "사과");
}

#[test]
fn two_functions_are_two_scopes() {
assert_names(
"고치다(정수 사) { } main() { 사과는 2예요. }",
&["사", "사과"],
);
// a header without a body (외부) keeps its parameters to itself
assert_names(
"외부 정수를 반환하는 abs(정수 사) main() { 사과는 2예요. }",
&["사", "사과"],
);
}

#[test]
fn a_rest_that_is_no_name_particle_is_no_conflict() {
// 득 is no particle, 로 and 의 only follow a type, 과일 only starts with one
assert_names("가는 1이에요. 가득은 2예요.", &["가", "가득"]);
assert_names(
"사는 1이에요. 사로는 2예요. 사의는 3이에요. 사과일은 4예요.",
&["사", "사로", "사의", "사과일"],
);
assert_names("고치다(정수 가, 정수 가득) { }", &["가", "가득"]);
}

#[test]
fn no_identifier_left_in_the_prototype() {
let tokens = pre(include_str!("../../../Proto.kr"));
Expand Down
Loading