tokenizer を整理

Author: senrust

.gitignore

@@ -1 +1,2 @@
 /target
+*.s

.vscode/launch.json

@@ -4,6 +4,25 @@
     // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug unit tests in library 'toy_compiler'",
+            "cargo": {
+                "args": [
+                    "test",
+                    "--no-run",
+                    "--lib",
+                    "--package=toy_compiler"
+                ],
+                "filter": {
+                    "name": "toy_compiler",
+                    "kind": "lib"
+                }
+            },
+            "args": [],
+            "cwd": "${workspaceFolder}"
+        },
         {
             "type": "lldb",
             "request": "launch",

src/compiler.rs

@@ -0,0 +1,38 @@
+use std::process::exit;
+use super::tokenizer::*;
+
+pub fn error_exit(error_text: &str) -> ! {
+    println!("{}", error_text);
+    exit(1);
+}
+
+pub fn expect_number(token_list: &mut TokenList) -> i32 {
+    let token = token_list.pop_head();
+    match token {
+        Some(valid_token) => {
+            match valid_token.token_kind {
+                TokenKind::Number(num) => {return num;},
+                _ => {error_exit("expect number token");}
+            }
+        },
+        None => {
+            error_exit("expect number token");
+        },
+    }
+}
+
+pub fn expect_operation(token_list: &mut TokenList) -> TokenKind {
+    let token = token_list.pop_head();
+    match token {
+        Some(valid_token) => {
+            match valid_token.token_kind {
+                TokenKind::Add => {return TokenKind::Add;},
+                TokenKind::Sub => {return TokenKind::Sub;},
+                _ => {error_exit("expect operation token");}
+            }
+        },
+        None => {
+            error_exit("expect operation token");
+        },
+    }
+}

src/lib.rs

@@ -0,0 +1,31 @@
+pub mod tokenizer;
+pub mod compiler;
+
+#[cfg(test)]
+mod tests {
+    use std::process::Command;
+    use super::tokenizer::*;
+    use super::compiler::*;
+
+  #[test]
+  fn binary_test() {
+        let status = Command::new("sh")
+        .arg("-c")
+        .arg("./a.out")
+        .status()
+        .expect("failed to execute mkdir").code().unwrap();
+    assert_eq!(status, 10);
+  }
+
+  #[test]
+  fn tokenizer_test() {
+    let input_text = "18 + 21 - 8";
+    let mut token_list = text_tokenizer(input_text);
+    assert_eq!(expect_number(&mut token_list), 18);
+    assert_eq!(expect_operation(&mut token_list), TokenKind::Add);
+    assert_eq!(expect_number(&mut token_list), 21);
+    assert_eq!(expect_operation(&mut token_list), TokenKind::Sub);
+    assert_eq!(expect_number(&mut token_list), 8);
+    assert!(token_list.is_empty());
+  }
+}

src/main.rs

@@ -1,8 +1,8 @@
 use std::fs;
 use std::io::{BufWriter, Write};
 
-mod tokenizer;
-use tokenizer::*;
+pub mod compiler;
+pub mod tokenizer;
 
 fn write_header<T: Write>(buf:&mut T) {
     writeln!(buf, ".intel_syntax noprefix").unwrap();
@@ -16,40 +16,8 @@ fn write_footer<T: Write>(buf:&mut T) {
 }
 
 fn main() {
-
-    let input_text = "18 + 21 - 8";
-    let mut tokens = text_tokenizer(input_text);
-    let token = tokens.pop_head().unwrap();
-    assert_eq!(token.token_kind, TokenKind::Number(18));
-    let token = tokens.pop_head().unwrap();
-    assert_eq!(token.token_kind, TokenKind::Add);
-    let token = tokens.pop_head().unwrap();
-    assert_eq!(token.token_kind, TokenKind::Number(21));
-    let token = tokens.pop_head().unwrap();
-    assert_eq!(token.token_kind, TokenKind::Sub);
-    let token = tokens.pop_head().unwrap();
-    assert_eq!(token.token_kind, TokenKind::Number(8));
-    let token = tokens.pop_head();
-    assert!(token.is_none());
-
-
     let mut file = BufWriter::new(fs::File::create("tmp.s").unwrap());
     write_header(&mut file);
     writeln!(file, "    mov rax, 10").unwrap();
     write_footer(&mut file);
 }
-
-#[cfg(test)]
-mod tests {
-    use std::process::Command;
-
-  #[test]
-  fn binary_test() {
-        let status = Command::new("sh")
-        .arg("-c")
-        .arg("./a.out")
-        .status()
-        .expect("failed to execute mkdir").code().unwrap();
-    assert_eq!(status, 10);
-  }
-}

src/tokenizer.rs

@@ -1,4 +1,4 @@
-use std::collections::VecDeque;
+use std::{collections::VecDeque, iter::FromIterator};
 
 #[derive(Debug,PartialEq,Eq)] 
 pub enum TokenKind {
@@ -11,13 +11,15 @@ pub enum TokenKind {
 #[derive(Debug)]
 pub struct Token {
     pub token_kind: TokenKind,
+    pub token_pos: i32,
     next: Option<Box<Token>>,
 }
 
 impl Token {
-    fn new() -> Token {
+    fn new(token_pos: i32) -> Token {
         Token{
             token_kind: TokenKind::InvalidToken,
+            token_pos,
             next: None,
         }
     }
@@ -37,26 +39,48 @@ impl TokenList {
 
     pub fn pop_head(&mut self) -> Option<Box<Token>> {
         if let Some(mut token) = self.head.take() {
-            self.head= token.next.take();
+            self.head = token.next.take();
             return Some(token);
         } else {
             return None;
         }
     }
+
+    pub fn is_empty(&mut self) -> bool {
+        self.head.is_none()
+    }
+}
+
+impl Drop for TokenList {
+    fn drop(&mut self) {
+        let mut token = self.head.take();
+        loop {
+            match token {
+                Some(mut valid_token) => {
+                    let next_token = valid_token.next.take();
+                    token = next_token;
+                },
+                None => {break;},
+            }
+        }
+    }
 }
 
+// 入力テキストのトークン連結リストを作成する
 pub fn text_tokenizer(text: &str) -> TokenList {
-    let mut char_queue = text.chars().collect::<VecDeque<char>>();
+    let mut char_queue = VecDeque::from_iter(text.chars());
     let mut tokenlist = TokenList::new();
     let mut current_token = &mut tokenlist.head;
+    let mut token_pos = -1;
 
     while !char_queue.is_empty() {
         let ch = char_queue.pop_front().unwrap();
+        token_pos += 1;
         if ch == ' ' {
             continue;
         }
 
-        let mut new_token = Token::new();
+        let mut new_token = Token::new(token_pos);
 
         if ch == '+' {
             new_token.token_kind = TokenKind::Add;
@@ -66,6 +90,7 @@ pub fn text_tokenizer(text: &str) -> TokenList {
             new_token.token_kind = TokenKind::Sub;
         }
 
+        // 数字処理の場合
         if ch.is_digit(10) {
             let mut num: i32 = ch.to_digit(10).unwrap() as i32;
             loop {
@@ -83,7 +108,7 @@ pub fn text_tokenizer(text: &str) -> TokenList {
             }
             new_token.token_kind = TokenKind::Number(num);
         }
-        
+       
         match current_token {
             Some(token) => {
                 token.next = Some(Box::new(new_token));