Chapter 7

src/lexer/token.rs

@@ -0,0 +1,64 @@
+use crate::misc::CodePos;
+
+#[allow(dead_code)]
+#[derive(Debug, Clone, PartialEq)]
+#[rustfmt::skip]
+pub enum TokenType {
+    // Single-character tokens
+    LeftParen, RightParen, LeftBrace, RightBrace,
+    Comma, Dot, Minus, Plus, Semicolon, Slash, Star,
+
+    // One or two character tokens
+    Bang, BangEqual,
+    Equal, EqualEqual,
+    Greater, GreaterEqual,
+    Less, LessEqual,
+
+    // Literals
+    Identifier(String),
+	String(String),
+	Number(f64),
+
+    // Keywords
+    And, Class, Else, False, Fun, For, If, Nil, Or,
+    Print, Return, Super, This, True, Var, While,
+
+    EOF
+}
+
+pub struct Token {
+    token_type: TokenType,
+    lexeme: String,
+
+    code_pos: CodePos,
+}
+
+impl Token {
+    pub fn new(token_type: TokenType, lexeme: String, pos: CodePos) -> Self {
+        Token {
+            token_type,
+            lexeme: lexeme,
+            code_pos: pos,
+        }
+    }
+
+    pub fn token_type(&self) -> &TokenType {
+        &self.token_type
+    }
+
+    pub fn code_pos(&self) -> CodePos {
+        self.code_pos
+    }
+}
+
+impl std::fmt::Debug for Token {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "<{:?}> (\"{}\")", self.token_type, self.lexeme)
+    }
+}
+
+impl std::fmt::Display for Token {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "<{:?}>", self.token_type)
+    }
+}