refactor: custom lexer #437
Conversation
| @@ -24,6 +24,7 @@ biome_rowan = "0.5.7" | |||
| biome_string_case = "0.5.8" | |||
| bpaf = { version = "0.9.15", features = ["derive"] } | |||
| crossbeam = "0.8.4" | |||
| enum-iterator = "2.1.0" | |||
There was a problem hiding this comment.
wir haben schon strum, denke mal das macht das selbe
| let mut ends_with_semicolon = false; | ||
|
|
||
| // Iterate through tokens in reverse to find the last non-whitespace token | ||
| for idx in (0..lexed.len()).rev() { |
There was a problem hiding this comment.
wie wär's hier mit matches(iter.filter(..).next_back(), Some(semi)) ?
|
|
||
| /// Returns an iterator over token kinds | ||
| pub fn tokens(&self) -> impl Iterator<Item = SyntaxKind> + '_ { | ||
| (0..self.len()).map(move |i| self.kind(i)) |
There was a problem hiding this comment.
self.kind.iter().copied() ?
|
|
||
| /// Returns the kind of token at the given index | ||
| pub fn kind(&self, idx: usize) -> SyntaxKind { | ||
| assert!(idx < self.len()); |
There was a problem hiding this comment.
willst du hier noch ne message anfügen?
ansonsten wär's vllt. besser, einfach den access in 53 panicen zu lassen, dann kriegt man zumindest eine index-out-of-bounds meldung, oder?
| .collect() | ||
| } | ||
|
|
||
| pub(crate) fn text_range(&self, i: usize) -> std::ops::Range<usize> { |
There was a problem hiding this comment.
ich glaube in allen fällen wird die std::ops::Range<usize> zu einer TextRange gemapped, vllt. dann besser die logic in range(..) packen?
| fn range_text(&self, r: std::ops::Range<usize>) -> &str { | ||
| assert!(r.start < r.end && r.end <= self.len()); | ||
| let lo = self.start[r.start] as usize; | ||
| let hi = self.start[r.end] as usize; | ||
| &self.text[lo..hi] | ||
| } |
There was a problem hiding this comment.
wenn ich richtig sehe wird das hier nur in text genutzt, vllt. die logik dann da rein packen statt einer indirection?
tokenizercrate that turns a string into simple tokenslexer+lexer_codegenthat uses the tokeniser to lex into a newSyntaxKindenumthe new implementation is
LineEndingvariant comes with a count)in a follow-up, we will be able to:
todos: