Cargo.lock

@@ -1,16 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-[[package]]
-name = "lazy_static"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "rust_lox"
 version = "0.1.0"
-dependencies = [
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
 
-[metadata]
-"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

Cargo.toml

@@ -5,4 +5,3 @@ authors = ["Sander Hautvast <shautvast@gmail.com>"]
 edition = "2018"
 
 [dependencies]
-lazy_static = "1.4.0"

README.md

@@ -1,4 +1,3 @@
-* Trying to learn rust as well as writing interpreters.
+Trying to learn rust as well as writing interpreters.
-* see https://www.craftinginterpreters.com/
 
-* the repo contains every iteration in a separate commit, so you can follow the development if you check them out individually
+see https://www.craftinginterpreters.com/

src/expression.rs

@@ -1,52 +0,0 @@
-use crate::tokens::{Token, Value};
-
-pub trait Visitor<R> {
-    fn visit_expr(&mut self, expr: &Expr) -> R;
-}
-
-#[derive(Debug, PartialOrd, PartialEq)]
-pub enum Expr {
-    Binary(Box<Expr>, Token, Box<Expr>),
-    Grouping(Box<Expr>),
-    Literal(Value),
-    Unary(Token, Box<Expr>),
-}
-
-pub struct AstPrinter {}
-
-impl Visitor<String> for AstPrinter {
-    fn visit_expr(&mut self, expr: &Expr) -> String {
-        return match expr {
-            Expr::Binary(left, operator, right) => {
-                self.parenthesize(&operator.lexeme, &[left, right])
-            }
-            Expr::Grouping(expression) => {
-                self.parenthesize("group", &[expression])
-            }
-            Expr::Literal(value) => {
-                format!("{:?}", value)
-            }
-            Expr::Unary(operator, right) => {
-                self.parenthesize(&operator.lexeme, &[right])
-            }
-        };
-    }
-}
-
-impl AstPrinter {
-    fn parenthesize(&mut self, name: &str, expressions: &[&Expr]) -> String {
-        let mut buf = String::from("(");
-        buf.push_str(name);
-        buf.push_str(" ");
-
-        let mut index = 0;
-        for expr in expressions {
-            if index > 0 { buf.push_str(" "); }
-            buf.push_str(&self.visit_expr(expr));
-            index += 1;
-        }
-
-        buf.push_str(")");
-        buf
-    }
-}

src/keywords.rs

@@ -1,27 +0,0 @@
-use std::collections::HashMap;
-
-use crate::tokens::TokenType;
-use crate::tokens::TokenType::*;
-
-lazy_static! {
-pub static ref KEYWORDS: HashMap<&'static str, TokenType> = {
-        let mut keywords = HashMap::new();
-        keywords.insert("and", AND);
-        keywords.insert("class", CLASS);
-        keywords.insert("else", ELSE);
-        keywords.insert("false", FALSE);
-        keywords.insert("for", FOR);
-        keywords.insert("fun", FUN);
-        keywords.insert("if", IF);
-        keywords.insert("nil", NIL);
-        keywords.insert("or", OR);
-        keywords.insert("print", PRINT);
-        keywords.insert("return", RETURN);
-        keywords.insert("super", SUPER);
-        keywords.insert("this", THIS);
-        keywords.insert("true", TRUE);
-        keywords.insert("var", VAR);
-        keywords.insert("while", WHILE);
-        keywords
-    };
-}

src/main.rs

@@ -1,22 +1,10 @@
-#[macro_use]
-extern crate lazy_static;
-
 use std::env;
 use std::fs::File;
 use std::io::{self, BufRead, Read, Write};
 use std::process;
 
-use crate::expression::{Visitor, AstPrinter};
-
 mod scanner;
 mod tokens;
-mod keywords;
-mod expression;
-mod parser;
-
-#[cfg(test)]
-mod scanner_tests;
-mod parser_tests;
 
 /// main
 /// no arguments: run interactively
@@ -89,8 +77,9 @@ fn run_prompt() {
 fn run(source: String) -> Result<&'static str, &'static str> {
     return match scanner::scan_tokens(source.as_str()) {
         Ok(tokens) => {
-            let expr = parser::parse(tokens);
+            for token in tokens {
-            println!("{:?}", AstPrinter {}.visit_expr(&expr));
+                println!("{:?}", token);
+            }
             Ok("Ok")
         }
         Err(code) => {

src/parser.rs

@@ -1,157 +0,0 @@
-use crate::expression::Expr;
-use crate::expression::Expr::*;
-use crate::tokens::{Token, TokenType};
-use crate::tokens::TokenType::*;
-use crate::tokens::Value::*;
-
-pub fn parse(tokens: Vec<Token>) -> Expr {
-    Parser::new(tokens).parse()
-}
-
-struct Parser {
-    tokens: Vec<Token>,
-    current: usize,
-}
-
-impl Parser {
-    fn new(tokens: Vec<Token>) -> Parser {
-        Parser { tokens, current: 0 }
-    }
-
-    fn parse(&mut self) -> Expr {
-        self.expression()
-    }
-
-    fn expression(&mut self) -> Expr {
-        self.equality()
-    }
-
-    fn equality(&mut self) -> Expr {
-        let mut expr = self.comparison();
-
-        while self.match_token(&[BANGEQUAL, EQUALEQUAL]) {
-            let operator = self.previous();
-            let right = self.comparison();
-            expr = Binary(Box::new(expr), operator, Box::new(right));
-        }
-
-        expr
-    }
-
-    fn comparison(&mut self) -> Expr {
-        let mut expr = self.addition();
-
-        while self.match_token(&[GREATER, GREATEREQUAL, LESS, LESSEQUAL]) {
-            let operator = self.previous();
-            let right = self.addition();
-            expr = Binary(Box::new(expr), operator, Box::new(right));
-        }
-
-        expr
-    }
-
-    fn match_token(&mut self, tokens: &[TokenType]) -> bool {
-        for token in tokens {
-            if self.check(*token) {
-                self.advance();
-                return true;
-            }
-        }
-
-        false
-    }
-
-    fn check(&self, token_type: TokenType) -> bool {
-        return if self.is_at_end() {
-            false
-        } else {
-            self.peek().token_type == token_type
-        };
-    }
-
-    fn peek(&self) -> Token {
-        return self.tokens[self.current].clone();
-    }
-
-    fn advance(&mut self) -> Token {
-        if !self.is_at_end() {
-            self.current += 1;
-        }
-        self.previous()
-    }
-
-    fn is_at_end(&self) -> bool {
-        self.peek().token_type == EOF
-    }
-
-    fn previous(&self) -> Token {
-        self.tokens[self.current - 1].clone()
-    }
-
-    fn addition(&mut self) -> Expr {
-        let mut expr = self.multiplication();
-
-        while self.match_token(&[MINUS, PLUS]) {
-            let operator = self.previous();
-            let right = self.multiplication();
-            expr = Binary(Box::new(expr), operator, Box::new(right));
-        }
-
-        expr
-    }
-
-    fn multiplication(&mut self) -> Expr {
-        let mut expr = self.unary();
-
-        while self.match_token(&[SLASH, STAR]) {
-            let operator = self.previous();
-            let right = self.unary();
-            expr = Binary(Box::new(expr), operator, Box::new(right));
-        }
-
-        return expr;
-    }
-
-    fn unary(&mut self) -> Expr {
-        if self.match_token(&[BANG, MINUS]) {
-            let operator = self.previous();
-            let right = self.unary();
-            return Unary(operator, Box::new(right));
-        }
-
-        return self.primary();
-    }
-
-    fn primary(&mut self) -> Expr {
-        if self.match_token(&[FALSE]) {
-            return Literal(Boolean(false));
-        }
-        if self.match_token(&[TRUE]) {
-            return Literal(Boolean(true));
-        }
-
-        if self.match_token(&[NIL]) {
-            return Literal(None);
-        }
-        if self.match_token(&[NUMBER, STRING]) {
-            return Literal(self.previous().literal);
-        }
-
-        if self.match_token(&[LEFTPAREN]) {
-            let expr = self.expression();
-            self.consume_token(RIGHTPAREN, "Expect ')' after expression.");
-            return Grouping(Box::new(expr));
-        } else {
-            Literal(None)
-        }
-    }
-
-    fn consume_token(&mut self, token_type: TokenType, _message: &str) -> Token {
-        if self.check(token_type) {
-            return self.advance();
-        }
-
-        panic!()
-    }
-}
-

src/parser_tests.rs

@@ -1,30 +0,0 @@
-use crate::expression::Expr::Binary;
-use crate::expression::Expr::Literal;
-use crate::parser::parse;
-use crate::scanner::scan_tokens;
-use crate::tokens::Token;
-use crate::tokens::TokenType::PLUS;
-use crate::tokens::Value::{None,Numeric};
-
-#[test]
-fn test_scan_empty_source() {
-    let tokens = scan_tokens("").unwrap();
-    let expression = parse(tokens);
-
-    assert_eq!(expression, Literal(None));
-}
-
-#[test]
-fn test_scan_arithmetic() {
-    let tokens = scan_tokens("1+1").unwrap();
-    let expression = parse(tokens);
-
-    assert_eq!(expression, Binary(Box::new(Literal(Numeric(1.0))),
-                                  Token {
-                                      token_type: PLUS,
-                                      lexeme: String::from("+"),
-                                      literal: None,
-                                      line: 1,
-                                  },
-                                  Box::new(Literal(Numeric(1.0)))));
-}

src/scanner.rs

@@ -1,5 +1,4 @@
-use crate::keywords::KEYWORDS;
+use crate::tokens::{Token, TokenType};
-use crate::tokens::{Token, TokenType, Value};
 use crate::tokens::TokenType::*;
 
 /// public function for scanning lox source
@@ -14,8 +13,8 @@ pub fn scan_tokens(source: &str) -> Result<Vec<Token>, &'static str> {
 
     scanner.tokens.push(Token {
         token_type: EOF,
-        lexeme: String::new(),
+        lexeme: "lexeme",
-        literal: Value::None,
+        literal: Box::new(""),
         line: scanner.line,
     });
 
@@ -32,7 +31,7 @@ struct Scanner<'a> {
     source: &'a str,
 
     // the tokens that will be the output of the scan function
-    tokens: Vec<Token>,
+    tokens: Vec<Token<'a>>,
 
     // start of unscanned source (updated after part of the source was scanned)
     start: usize,
@@ -100,72 +99,7 @@ impl Scanner<'_> {
             ' ' => {}
             '\t' => {}
             '\r' => {}
-            '\"' => self.string(),
+            _ => {}
-            _ => {
-                if next_char.is_digit(10) {
-                    self.number();
-                } else if is_alphabetic_or_underscore(next_char) {
-                    self.identifier();
-                } else {
-                    self.report_error(self.line, "unexpected character");
-                }
-            }
-        }
-    }
-
-    fn identifier(&mut self) {
-        while is_alphanumeric(self.peek(0)) {
-            self.advance();
-        }
-        let text = &self.source[self.start..self.current];
-        match KEYWORDS.get(text) {
-            Some(token_type) => {
-                self.add_token(*token_type);
-            }
-            None => {
-                self.add_token(TokenType::IDENTIFIER);
-            }
-        }
-    }
-
-    /// handle number literals
-   /// advances while characters are considered part of the number
-   /// finally adds a number token to the list.
-    fn number(&mut self) {
-        while self.peek(0).is_digit(10) {
-            self.advance();
-        }
-
-        if self.peek(0) == '.' && self.peek(1).is_digit(10) {
-            self.advance();
-
-            while self.peek(0).is_digit(10) {
-                self.advance();
-            }
-        }
-        let value: f64 = self.source[self.start..self.current].parse().expect("not a number");
-
-        self.add_token_literal(NUMBER, Value::Numeric(value));
-    }
-
-    /// handle string literals
-    /// advances until a terminating double quote is found and then adds the string token to the list
-    /// raises an interpreter error when the double-quote is not found and the end of the source has been reached
-    fn string(&mut self) {
-        while self.peek(0) != '\"' && !self.is_at_end() {
-            if self.peek(0) == '\n' {
-                self.line += 1;
-            }
-            self.advance();
-        }
-
-        if self.is_at_end() {
-            self.report_error(self.line, "unterminated string");
-        } else {
-            self.advance();
-
-            let value = String::from(&self.source[self.start + 1..self.current - 1]);
-            self.add_token_literal(STRING, Value::Text(value));
         }
     }
 
@@ -178,14 +112,7 @@ impl Scanner<'_> {
     /// adds a token of the given type
     fn add_token(&mut self, token_type: TokenType) {
         let text = &self.source[self.start..self.current];
-        let token = Token { token_type: token_type, lexeme: String::from(text), literal: Value::None, line: self.line };
+        let token = Token { token_type: token_type, lexeme: text, literal: Box::new(""), line: self.line };
-        self.tokens.push(token);
-    }
-
-    /// adds a token of the given type and content
-    fn add_token_literal(&mut self, token_type: TokenType, literal: Value) {
-        let text = &self.source[self.start..self.current];
-        let token = Token { token_type: token_type, lexeme: String::from(text), literal, line: self.line };
         self.tokens.push(token);
     }
 
@@ -218,19 +145,4 @@ impl Scanner<'_> {
         self.current += 1;
         true
     }
-
-    /// prints the error and sets the flag
-    pub fn report_error(&mut self, line: usize, message: &str) {
-        self.error_occured = true;
-        println!("[line {} ] Error {} ", line, message);
-    }
-}
-
-
-fn is_alphabetic_or_underscore(c: char) -> bool {
-    c.is_alphabetic() || c == '_'
-}
-
-fn is_alphanumeric(c: char) -> bool {
-    is_alphabetic_or_underscore(c) || c.is_digit(10)
 }

src/scanner_tests.rs

@@ -1,105 +0,0 @@
-#[cfg(test)]
-use crate::scanner::scan_tokens;
-use crate::tokens::TokenType::*;
-use crate::tokens::Value::{Numeric, Text};
-
-#[test]
-fn test_scan_empty_source() {
-    let tokens = scan_tokens("").unwrap();
-    assert_eq!(tokens.len(), 1);
-
-    let token = tokens.get(0).unwrap();
-    assert_eq!(token.token_type, EOF);
-    assert_eq!(token.line, 1);
-}
-
-#[test]
-fn test_scan_single_char_tokens() {
-    let tokens = scan_tokens(">").unwrap();
-    assert_eq!(tokens.len(), 2);
-
-    let token = tokens.get(0).unwrap();
-    assert_eq!(token.token_type, GREATER);
-    assert_eq!(token.lexeme, ">");
-
-    let token = tokens.get(1).unwrap();
-    assert_eq!(token.token_type, EOF);
-}
-
-#[test]
-fn test_scan_double_char_tokens() {
-    let tokens = scan_tokens(">=").unwrap();
-    assert_eq!(tokens.len(), 2);
-
-    let token = tokens.get(0).unwrap();
-    assert_eq!(token.token_type, GREATEREQUAL);
-    assert_eq!(token.lexeme, ">=");
-}
-
-
-#[test]
-fn test_scan_string_literals() {
-    let tokens = scan_tokens("\"hello world\"").unwrap();
-    assert_eq!(tokens.len(), 2);
-
-    let token = tokens.get(0).unwrap();
-    assert_eq!(token.token_type, STRING);
-    assert_eq!(token.lexeme, "\"hello world\"");
-    match token.literal.clone() {
-        Text(value) => {
-            assert_eq!(value, "hello world");
-        }
-        _ => { assert_eq!(true,false, "token value != hello world") }
-    }
-}
-
-#[test]
-fn test_scan_numeric_literals() {
-    let tokens = scan_tokens("0.1").unwrap();
-    assert_eq!(tokens.len(), 2);
-
-    let token = tokens.get(0).unwrap();
-    assert_eq!(token.token_type, NUMBER);
-    assert_eq!(token.lexeme, "0.1");
-    match token.literal {
-        Numeric(value) => {
-            assert_eq!(value, 0.1);
-        }
-        _ => { assert_eq!(true, false, "token value != 0.1") }
-    }
-}
-
-#[test]
-fn test_keywords() {
-    let tokens = scan_tokens("fun").unwrap();
-    assert_eq!(tokens.len(), 2);
-
-    let token = tokens.get(0).unwrap();
-    assert_eq!(token.token_type, FUN);
-}
-
-#[test]
-fn test_identifiers() {
-    let tokens = scan_tokens("a").unwrap();
-    assert_eq!(tokens.len(), 2);
-
-    let token = tokens.get(0).unwrap();
-    assert_eq!(token.token_type, IDENTIFIER);
-}
-
-#[test]
-fn test_expression() {
-    let tokens = scan_tokens("if a == 1 {b=\"hello world\"}").unwrap();
-    assert_eq!(tokens.len(), 10);
-
-    assert_eq!(tokens.get(0).unwrap().token_type, IF);
-    assert_eq!(tokens.get(1).unwrap().token_type, IDENTIFIER);
-    assert_eq!(tokens.get(2).unwrap().token_type, EQUALEQUAL);
-    assert_eq!(tokens.get(3).unwrap().token_type, NUMBER);
-    assert_eq!(tokens.get(4).unwrap().token_type, LEFTBRACE);
-    assert_eq!(tokens.get(5).unwrap().token_type, IDENTIFIER);
-    assert_eq!(tokens.get(6).unwrap().token_type, EQUAL);
-    assert_eq!(tokens.get(7).unwrap().token_type, STRING);
-    assert_eq!(tokens.get(8).unwrap().token_type, RIGHTBRACE);
-    assert_eq!(tokens.get(9).unwrap().token_type, EOF);
-}

src/tokens.rs

@@ -1,95 +1,60 @@
+use std::any::Any;
 use std::fmt;
 
-#[derive(Clone, PartialOrd, PartialEq)]
+/// struct that contains a single token
-pub enum Value {
+pub struct Token<'a> {
-    Text(String),
+    // the type
-    Numeric(f64),
+    pub lexeme: &'a str,
-    Boolean(bool),
+
-    None,
+    // the actual part of the code that resulted in this token
+    pub literal: Box<dyn Any>,
+
+    // numeric (ie 1,2, 1.0 etc) and alphanumeric (any quoted text) values
+    pub line: usize,
+
+    // the line that contains the code for this token instance
+    pub token_type: TokenType,
 }
 
-impl fmt::Debug for Value {
+impl fmt::Debug for Token<'_> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
+        let lit = match self.literal.downcast_ref::<String>() {
-            Value::Text(value) => {
+            Some(as_string) => {
-                write!(f, "{}", value.to_string())
+                as_string.to_string()
             }
-            Value::Numeric(value) => {
+            None => {
-                write!(f, "{}", value)
+                format!("{:?}", self.literal)
             }
-            Value::Boolean(value) => {
+        };
-                write!(f, "{}", value)
+
-            }
+        write!(f, "Token [ type: {:?}, lexeme: {}, literal: {}, line: {} ]", self.token_type, self.lexeme, lit, self.line)
-            Value::None => {
-                write!(f, "Nil")
-            }
-        }
     }
 }
 
-/// struct that contains a single token
+#[derive(Debug, Clone, Copy)]
-#[derive(Debug, Clone, PartialOrd, PartialEq)]
-pub struct Token {
-    // the type
-    pub token_type: TokenType,
-
-    // the actual part of the code that resulted in this token
-    pub lexeme: String,
-
-    // numeric (ie 1,2, 1.0 etc) and alphanumeric (any quoted text) values
-    pub literal: Value,
-
-    // the line that contains the code for this token instance
-    pub line: usize,
-}
-
-#[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd)]
 pub enum TokenType {
     // Single-character tokens.
-    LEFTPAREN,
+    LEFTPAREN,  // (
-    RIGHTPAREN,
+    RIGHTPAREN, // )
-    LEFTBRACE,
+    LEFTBRACE,  // [
-    RIGHTBRACE,
+    RIGHTBRACE, // ]
-    COMMA,
+    COMMA,      // ,
-    DOT,
+    DOT,        // .
-    MINUS,
+    MINUS,      // -
-    PLUS,
+    PLUS,       // +
-    SEMICOLON,
+    SEMICOLON,  // ;
-    STAR,
+    STAR,       // *
-    SLASH,
+    SLASH,      // /
 
     // One or two character tokens.
-    BANG,
+    BANG,           // !
-    BANGEQUAL,
+    BANGEQUAL,      // !=
-    EQUAL,
+    EQUAL,          // =
-    EQUALEQUAL,
+    EQUALEQUAL,     // ==
-    GREATER,
+    GREATER,        // >
-    GREATEREQUAL,
+    GREATEREQUAL,   // >=
-    LESS,
+    LESS,           // <
-    LESSEQUAL,
+    LESSEQUAL,      // <=
 
-    // Literals.
+    EOF         // end of file
-    STRING,
-    NUMBER,
-    IDENTIFIER,
-
-    // Keywords.
-    AND,
-    CLASS,
-    ELSE,
-    FALSE,
-    FUN,
-    FOR,
-    IF,
-    NIL,
-    OR,
-    PRINT,
-    RETURN,
-    SUPER,
-    THIS,
-    TRUE,
-    VAR,
-    WHILE,
-
-    EOF,         // end of file
 }