Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 24 additions & 4 deletions scripts/missing.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
# on any other branch: every issue, only what is missing
# --all every issue, whatever the branch
# issue... these issues, passing checks included
# With issues given (or found from the branch), each MISS of a checks.sh also shows its program and
# the whole stderr and exit code of the compiler on it.
# No environment variable. Exit code: 0 when nothing is missing, 1 otherwise, 2 on a wrong argument.
# Example: scripts/missing.sh 33 9
#
Expand All @@ -16,7 +18,7 @@
# src/issue_specs.rs `mod iN`: #[ignore] tests of what only the internals can show

if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//'
sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//'
exit 0
fi

Expand Down Expand Up @@ -67,6 +69,9 @@ WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
RESULTS="$WORK/results"
: > "$RESULTS"
DETAILS="$WORK/details" # one file per MISS of a checks.sh: <issue>.<label>
mkdir "$DETAILS"
verbose=$([ "$mode" = some ] && echo 1)

issue_of() { # issue_of <path under specs/>: the number of the issue
local base="${1#"$ROOT"/specs/}"
Expand All @@ -80,11 +85,23 @@ record() { # record <issue> <ok|MISS|ASK> <label> <reason>
# spec, sources to the repository

check() { # check <label> <helper> <args...>: one line of the report
local why
local why kr out code
if why="$("${@:2}" 2>&1)"; then
record "$ISSUE" ok "$1" ""
else
record "$ISSUE" MISS "$1" "$(tail -n 1 <<< "$why")"
[ -n "$verbose" ] || return 0
# the helper ran in a subshell: compile again to keep the program and its whole stderr
for kr in "${@:3}"; do
[[ "$kr" == *.kr && -f "$kr" ]] || continue
out="$("$COMPILER" "$kr" 2>&1 > /dev/null)"
code=$?
{
cat -n "$kr"
echo "--- stderr (exit $code):"
printf '%s\n' "${out:-(empty)}"
} > "$DETAILS/$ISSUE.${1//\//_}"
done
fi
}
manual() { # manual <text>: a decision or a task no test can see, always listed
Expand Down Expand Up @@ -219,7 +236,6 @@ fi

# --- report

verbose=$([ "$mode" = some ] && echo 1)
total_ok=0
total_miss=0
for dir in "${dirs[@]}"; do
Expand All @@ -236,7 +252,11 @@ for dir in "${dirs[@]}"; do
while IFS=$'\t' read -r _ status label reason; do
case "$status" in
ok) [ -z "$verbose" ] || echo " ok $label" ;;
MISS) echo " MISS $label${reason:+: ${reason:0:200}}" ;;
MISS)
echo " MISS $label${reason:+: ${reason:0:200}}"
details="$DETAILS/$n.${label//\//_}"
[ ! -f "$details" ] || sed 's/^/ /' "$details"
;;
ASK) echo " ASK $label" ;;
esac
done < "$WORK/one"
Expand Down
16 changes: 15 additions & 1 deletion src/hangeul/syllable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,25 @@ impl Syllable {
})
}

pub fn to_char(self) -> char {
pub fn char_from_number(num: i64) -> Option<char> {
const DIGITS: [char; 10] = ['영', '일', '이', '삼', '사', '오', '육', '칠', '팔', '구'];
return Some(*DIGITS.get(num as usize)?);
}

#[allow(dead_code)]
pub fn from_number(num: i64) -> Option<Syllable> {
Self::from_char(Self::char_from_number(num)?)
}

pub fn to_char(&self) -> char {
let code = 588 * self.initial as u32 + 28 * self.vowel as u32 + self.batchim as u32;
char::from_u32(FIRST + code).expect("jamo index out of range")
}

pub fn end_with_batchim(c: char) -> bool {
Self::from_char(c).is_some_and(|s| s.has_batchim())
}

pub fn has_batchim(self) -> bool {
self.batchim != 0
}
Expand Down
221 changes: 221 additions & 0 deletions src/parser/preparser/classify.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,221 @@
use super::super::parser_token::{Keyword, ParserToken, ParserTokenKind, Particle, Type};
use crate::hangeul::syllable::Syllable;
use crate::lexer::lexer_token::{LexerToken, LexerTokenKind, Operator, Punctuation};
use crate::lexer::span::Span;

use super::pre_parse_error::PreParseError;
use super::pre_parse_warning::{PreParseWarning, PreParseWarningKind};
use super::pre_parser::PreParser;
use super::words::{
expected_form, has_particule, is_bool, is_counter, is_keyword, is_name_particle, is_particle,
is_type,
};

/// One lexer token into parser tokens, before the declared names are known
impl PreParser<'_> {
/// One lexer token gives one parser token, or two for a split word (정수를 → 정수 + 를);
/// `prev` is the lexer token before it, if any
pub(super) fn transform_token(
&mut self,
token: &LexerToken,
prev: Option<&LexerTokenKind>,
) -> Result<Vec<ParserToken>, PreParseError> {
let kind = match &token.kind {
LexerTokenKind::HangulWord(word) => return self.hangul_word(word, token.span, prev),
LexerTokenKind::LatinWord(name) => ParserTokenKind::Name(name.clone()),
LexerTokenKind::Float(value) => ParserTokenKind::Float(*value),
LexerTokenKind::Int(value) => ParserTokenKind::Int(*value),
LexerTokenKind::Str(value) => ParserTokenKind::Str(value.clone()),
LexerTokenKind::Punctuation(punctuation) => ParserTokenKind::Punctuation(*punctuation),
LexerTokenKind::Operator(operator) => ParserTokenKind::Operator(*operator),
LexerTokenKind::Eof => ParserTokenKind::Eof,
};
Ok(vec![ParserToken::new(kind, token.span)])
}

/// After a number, a string, `)` or a Latin name, `word` is a particle alone (42를, 0이에요,
/// printf를); after a number, a warning when it does not fit the 받침 of the number read in
/// Sino-Korean. None when `word` is no particle there
fn check_solo_particule(
&mut self,
word: &str,
span: Span,
prev: Option<&LexerTokenKind>,
) -> Option<Vec<ParserToken>> {
let particle = is_particle(word)?;
match prev? {
// the 받침 of the last syllable read: the last digit, or 십 백 천… (영 for 0) when
// it is 0, which all have one like 영; (으)로 and 의 have no rule here
LexerTokenKind::Int(n) if is_name_particle(word).is_some() => {
let read = Syllable::char_from_number(n % 10)?.to_string();
self.check_batchim(&n.to_string(), &read, word, &particle, span);
}
LexerTokenKind::Int(_)
| LexerTokenKind::Float(_)
| LexerTokenKind::Str(_)
| LexerTokenKind::LatinWord(_) => {}
LexerTokenKind::Punctuation(Punctuation::RParen) => {
// the last thing read before the `)`s: (가 + 나)를 → 나, ((가 + 3))를 → 3
let tokens = self.tokens;
let last = tokens[..self.pos - 1]
.iter()
.rev()
.find(|t| t.kind != LexerTokenKind::Punctuation(Punctuation::RParen))
.map(|t| &t.kind);
match last {
Some(LexerTokenKind::Int(_)) => {
return self.check_solo_particule(word, span, last);
}
Some(LexerTokenKind::HangulWord(name)) if is_name_particle(word).is_some() => {
self.check_batchim(name, name, word, &particle, span);
}
_ => {}
}
}

_ => return None,
}
Some(vec![ParserToken::new(
ParserTokenKind::Particle(particle),
span,
)])
}

/// Classifies a Hangul word, in this order:
/// 1. the whole word is a keyword, a type, a boolean, a counter before a type or a verb form:
/// 반환하는, 정수, 참, 두 (정수), 더해서 (반환하는 is not 반환하 + 는)
/// 2. after a number, a string, `)` or a Latin name, the whole word is a particle: 42를,
/// 0이에요, printf를
/// 3. a keyword, a type or a boolean followed by a particle: 정수를, 값이에요, 10개예요, 참이면
/// 4. anything else is an identifier, left whole: 나를, 결과
fn hangul_word(
&mut self,
word: &str,
span: Span,
prev: Option<&LexerTokenKind>,
) -> Result<Vec<ParserToken>, PreParseError> {
let whole = |kind| Ok(vec![ParserToken::new(kind, span)]);

if let Some(keyword) = is_keyword(word) {
return whole(ParserTokenKind::Keyword(keyword));
}
if let Some(ty) = is_type(word) {
return whole(ParserTokenKind::Type(ty));
}
if let Some(value) = is_bool(word) {
return whole(ParserTokenKind::Bool(value));
}
if let Some(count) = is_counter(word)
&& self.type_follows()
{
return whole(ParserTokenKind::Counter(count));
}
if let Some((infinitive, ending)) = self.verbs.get(word) {
return whole(ParserTokenKind::Verb {
infinitive: infinitive.clone(),
ending: *ending,
});
}

if let Some(particle) = self.check_solo_particule(word, span, prev) {
return Ok(particle);
}

if let Some((stem, particle)) = has_particule(word) {
let stem_kind = is_keyword(stem)
.map(ParserTokenKind::Keyword)
.or_else(|| is_type(stem).map(ParserTokenKind::Type))
.or_else(|| is_bool(stem).map(ParserTokenKind::Bool));
if let Some(stem_kind) = stem_kind {
self.check_batchim(stem, stem, &word[stem.len()..], &particle, span);
// byte offsets, like Span: 정수를 0..9 → 정수 0..6, 를 6..9
let cut = span.start + stem.len();
return Ok(vec![
ParserToken::new(stem_kind, Span::new(span.start, cut)),
ParserToken::new(
ParserTokenKind::Particle(particle),
Span::new(cut, span.end),
),
]);
}
}

// not split here: 결과 is a name, not 결 + 과, and only the declared names can tell
whole(ParserTokenKind::Ident(word.to_string()))
}

/// A warning when `written`, the particle after `shown`, is not the form the 받침 of `read`
/// asks for: `shown` is the stem as written (42, 정수), `read` how it sounds (이, 정수)
pub(super) fn check_batchim(
&mut self,
shown: &str,
read: &str,
written: &str,
particle: &Particle,
span: Span,
) {
let expected = expected_form(read, particle);
let form = &expected[read.len()..];
if written != form {
self.warnings.push(PreParseWarning::new(
PreParseWarningKind::BadBatchim {
found: format!("{shown}{written}"),
expected: format!("{shown}{form}"),
},
span,
));
}
}

/// Whether the next lexer token starts a type: 정수, or 짧은 / 부호 (없는) before it. A counter
/// is only one before a type: 세 and 열 stay free as names
fn type_follows(&self) -> bool {
let LexerTokenKind::HangulWord(next) = &self.peek().kind else {
return false;
};
is_type(next).is_some() || next == "짧은" || next == "부호"
}

/// `부호 없는` is two lexer tokens for one keyword: reads both, gives one `Unsigned` spanning
/// them; None, reading nothing more, for anything else
pub(super) fn merge_unsigned(&mut self, token: &LexerToken) -> Option<ParserToken> {
let LexerTokenKind::HangulWord(first) = &token.kind else {
return None;
};
let LexerTokenKind::HangulWord(second) = &self.peek().kind else {
return None;
};
if first != "부호" || second != "없는" {
return None;
}
let end = self.bump().span.end;
Some(ParserToken::new(
ParserTokenKind::Keyword(Keyword::Unsigned),
Span::new(token.span.start, end),
))
}

/// `&` right after a type is `주소`: `정수&` → Type(Int) + Type(Address); since it becomes a
/// type itself, `정수&&` is two addresses. Before the declared names, so that `정수& 가)`
/// declares 가; anywhere else `&` stays the operator
pub(super) fn amp_after_a_type(
parsed: Vec<ParserToken>,
last: Option<&ParserToken>,
) -> Vec<ParserToken> {
match (parsed.as_slice(), last.map(|token| &token.kind)) {
(
[
ParserToken {
kind: ParserTokenKind::Operator(Operator::Amp),
span,
},
],
Some(ParserTokenKind::Type(_)),
) => vec![ParserToken::new(
ParserTokenKind::Type(Type::Address),
*span,
)],
_ => parsed,
}
}
}
3 changes: 3 additions & 0 deletions src/parser/preparser/mod.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
pub mod pre_parse_error;
pub mod pre_parse_warning;

mod classify;
mod names;
mod pre_parser;
mod verbs;
mod words;

#[cfg(test)]
Expand Down
Loading
Loading