minimal compiler and vm

src/chunk.rs

@@ -0,0 +1,78 @@
+use crate::opcode::{OP_CONSTANT, OP_RETURN, OP_NEGATE, OP_ADD, OP_SUBTRACT, OP_MULTIPLY, OP_DIVIDE};
+use crate::value::Value;
+
+pub struct Chunk {
+    name: String,
+    pub code: Vec<u16>,
+    pub constants: Vec<Value>,
+    lines: Vec<usize>,
+}
+
+impl Chunk {
+    pub fn new(name: &str) -> Chunk {
+        Chunk {
+            name: name.to_string(),
+            code: Vec::new(),
+            constants: vec![],
+            lines: vec![],
+        }
+    }
+
+    pub fn add(&mut self, byte: u16, line: usize) {
+        self.code.push(byte);
+        self.lines.push(line);
+    }
+
+    pub fn add_constant(&mut self, value: impl Into<Value>) -> usize {
+        self.constants.push(value.into());
+        self.constants.len() - 1
+    }
+
+    pub fn disassemble(&self) {
+        println!("== {} ==", self.name);
+        let mut offset = 0;
+        while offset < self.code.len() {
+            offset = self.disassemble_inst(offset);
+        }
+        println!("== {} ==", self.name);
+    }
+
+    fn disassemble_inst(&self, offset: usize) -> usize {
+        print!("{:04} ", offset);
+        if offset > 0 && self.lines[offset] == self.lines[offset - 1] {
+            print!("   | ");
+        } else {
+            print!("{:04} ", self.lines[offset]);
+        }
+        let instruction = self.code[offset];
+        match instruction {
+            OP_CONSTANT => self.constant_inst("OP_CONSTANT", offset),
+            OP_ADD => self.simple_inst("OP_ADD", offset),
+            OP_SUBTRACT => self.simple_inst("OP_SUBTRACT", offset),
+            OP_MULTIPLY => self.simple_inst("OP_MULTIPLY", offset),
+            OP_DIVIDE => self.simple_inst("OP_DIVIDE", offset),
+            OP_NEGATE => self.simple_inst("OP_NEGATE", offset),
+            OP_RETURN => self.simple_inst("OP_RETURN", offset),
+            _ => {
+                println!("Unknown instruction");
+                offset + 1
+            }
+        }
+    }
+
+    fn simple_inst(&self, name: &str, offset: usize) -> usize {
+        println!("{}", name);
+        offset + 1
+    }
+
+    fn constant_inst(&self, name: &str, offset: usize) -> usize {
+        let constant = self.code[offset + 1];
+        print!("{} {} ", name, constant);
+        self.print_value(&self.constants[constant as usize]);
+        offset + 2
+    }
+
+    fn print_value(&self, value: &Value) {
+        println!("'{:?}'", value);
+    }
+}

src/compiler.rs

@@ -0,0 +1,239 @@
+use crate::chunk::Chunk;
+use crate::opcode::{
+    OP_ADD, OP_CONSTANT, OP_DIVIDE, OP_MULTIPLY, OP_NEGATE, OP_RETURN, OP_SUBTRACT,
+};
+use crate::scanner::scan;
+use crate::tokens::TokenType::Print;
+use crate::tokens::{Token, TokenType};
+use crate::value::Value;
+use anyhow::anyhow;
+use std::collections::HashMap;
+use std::sync::LazyLock;
+
+pub fn compile(source: &str) -> anyhow::Result<Chunk> {
+    let tokens = scan(source);
+
+    let mut compiler = Compiler {
+        chunk: Chunk::new("main"),
+        previous_token: &tokens[0],
+        current_token: &tokens[0],
+        tokens: &tokens,
+        current: 0,
+        previous: 0,
+        had_error: false,
+    };
+    compiler.compile()
+}
+
+struct Compiler<'a> {
+    chunk: Chunk,
+    tokens: &'a Vec<Token>,
+    current: usize,
+    previous_token: &'a Token,
+    current_token: &'a Token,
+    previous: usize,
+    had_error: bool,
+}
+
+impl<'a> Compiler<'a> {
+    fn compile(mut self) -> anyhow::Result<Chunk> {
+        self.expression()?;
+        self.consume(TokenType::Eof, "Expect end of expression.")?;
+        self.emit_byte(OP_RETURN);
+        Ok(self.chunk)
+    }
+
+    fn advance(&mut self) -> anyhow::Result<()> {
+        if self.current < self.tokens.len() - 1 {
+            self.previous = self.current;
+            self.previous_token = &self.tokens[self.previous];
+            self.current += 1;
+            self.current_token = &self.tokens[self.current];
+        }
+        if let TokenType::Error = self.current_token.tokentype {
+            self.had_error = true;
+            Err(anyhow!(
+                "Error at {} on line {}",
+                self.current_token.lexeme,
+                self.current_token.line
+            ))
+        } else {
+            Ok(())
+        }
+    }
+
+    fn consume(&mut self, token_type: TokenType, message: &str) -> anyhow::Result<()> {
+        if token_type == self.current_token.tokentype {
+            self.advance()
+        } else {
+            Err(anyhow!("{}", message))
+        }
+    }
+
+    fn expression(&mut self) -> anyhow::Result<()> {
+        self.parse_precedence(PREC_ASSIGNMENT)?;
+        Ok(())
+    }
+
+    fn parse_precedence(&mut self, precedence: usize) -> anyhow::Result<()> {
+        self.advance()?;
+        let prefix_rule = get_rule(&self.previous_token.tokentype).prefix;
+        if let Some(prefix) = prefix_rule {
+            prefix(self)?;
+            while precedence <= get_rule(&self.current_token.tokentype).precedence {
+                self.advance()?;
+                let infix_rule = get_rule(&self.previous_token.tokentype).infix;
+                if let Some(infix) = infix_rule {
+                    infix(self)?;
+                }
+            }
+        } else {
+            return Err(anyhow!("Expect expression."));
+        }
+        Ok(())
+    }
+
+    fn emit_byte(&mut self, byte: u16) {
+        self.chunk.add(byte, self.previous_token.line);
+    }
+
+    fn emit_bytes(&mut self, b1: u16, b2: u16) {
+        self.emit_byte(b1);
+        self.emit_byte(b2);
+    }
+
+    fn emit_constant(&mut self, value: Value) {
+        let index = self.chunk.add_constant(value);
+        self.emit_bytes(OP_CONSTANT, index as u16);
+    }
+}
+
+type ParseFn = fn(&mut Compiler) -> anyhow::Result<()>;
+
+struct Rule {
+    prefix: Option<ParseFn>,
+    infix: Option<ParseFn>,
+    precedence: usize,
+}
+
+impl Rule {
+    fn new(prefix: Option<ParseFn>, infix: Option<ParseFn>, precedence: usize) -> Self {
+        Self {
+            prefix,
+            infix,
+            precedence,
+        }
+    }
+}
+
+fn number(s: &mut Compiler) -> anyhow::Result<()> {
+    let tt = s.previous_token.tokentype;
+    let value = s.previous_token.lexeme.clone();
+    s.emit_constant(match tt {
+        TokenType::Number => Value::F64(value.parse().unwrap()),
+        _ => unimplemented!(), // TODO numeric types
+    });
+    Ok(())
+}
+
+fn grouping(s: &mut Compiler) -> anyhow::Result<()> {
+    s.expression()?;
+    s.consume(TokenType::RightParen, "Expect ')' after expression.")
+}
+
+fn unary(s: &mut Compiler) -> anyhow::Result<()> {
+    let operator_type = s.previous_token.tokentype;
+
+    s.parse_precedence(PREC_UNARY)?;
+
+    match operator_type {
+        TokenType::Minus => {
+            s.emit_byte(OP_NEGATE);
+        }
+        _ => unimplemented!("unary other than minus"),
+    }
+    Ok(())
+}
+
+fn binary(s: &mut Compiler) -> anyhow::Result<()> {
+    let operator_type = &s.previous_token.tokentype;
+    let rule = get_rule(operator_type);
+    s.parse_precedence(rule.precedence + 1)?;
+    match operator_type {
+        TokenType::Plus => s.emit_byte(OP_ADD),
+        TokenType::Minus => s.emit_byte(OP_SUBTRACT),
+        TokenType::Star => s.emit_byte(OP_MULTIPLY),
+        TokenType::Slash => s.emit_byte(OP_DIVIDE),
+        _ => unimplemented!("binary other than plus, minus, star, slash"),
+    }
+    Ok(())
+}
+
+fn get_rule(operator_type: &TokenType) -> &'static Rule {
+    RULES.get(operator_type).unwrap()
+}
+
+static RULES: LazyLock<HashMap<TokenType, Rule>> = LazyLock::new(|| {
+    let mut rules: HashMap<TokenType, Rule> = HashMap::new();
+    rules.insert(
+        TokenType::LeftParen,
+        Rule::new(Some(binary), None, PREC_NONE),
+    );
+    rules.insert(TokenType::RightParen, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::LeftBrace, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::RightBrace, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::LeftBracket, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::RightBracket, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Comma, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Dot, Rule::new(None, None, PREC_NONE));
+    rules.insert(
+        TokenType::Minus,
+        Rule::new(Some(unary), Some(binary), PREC_TERM),
+    );
+    rules.insert(TokenType::Plus, Rule::new(None, Some(binary), PREC_TERM));
+    rules.insert(TokenType::Colon, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Slash, Rule::new(None, Some(binary), PREC_FACTOR));
+    rules.insert(TokenType::Star, Rule::new(None, Some(binary), PREC_FACTOR));
+    rules.insert(TokenType::Bang, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::BangEqual, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Equal, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::EqualEqual, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Greater, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::GreaterEqual, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Less, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::LessEqual, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Identifier, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::String, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Number, Rule::new(Some(number), None, PREC_NONE));
+    rules.insert(TokenType::And, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Fn, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Struct, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Else, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::False, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::True, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Or, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::While, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Print, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Return, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::Eof, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::U32Type, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::U64Type, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::I32Type, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::I64Type, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::DateType, Rule::new(None, None, PREC_NONE));
+    rules.insert(TokenType::StringType, Rule::new(None, None, PREC_NONE));
+
+    rules
+});
+
+const PREC_NONE: usize = 0;
+const PREC_ASSIGNMENT: usize = 1;
+const PREC_OR: usize = 2;
+const PREC_AND: usize = 3;
+const PREC_EQUALITY: usize = 4;
+const PREC_COMPARISON: usize = 5;
+const PREC_TERM: usize = 6;
+const PREC_FACTOR: usize = 7;
+const PREC_UNARY: usize = 8;
+const PREC_CALL: usize = 9;
+const PREC_PRIMARY: usize = 10;

src/keywords.rs

@@ -4,6 +4,19 @@ pub(crate) fn get_keyword(lexeme: &str) -> Option<TokenType> {
     match lexeme {
         "fn" => Some(TokenType::Fn),
         "struct" => Some(TokenType::Struct),
+        "u32" => Some(TokenType::U32Type),
+        "string" => Some(TokenType::StringType),
+        "date" => Some(TokenType::DateType),
+        "print" => Some(TokenType::Print),
+        "and" => Some(TokenType::And),
+        "else" => Some(TokenType::Else),
+        "false" => Some(TokenType::False),
+        "true" => Some(TokenType::True),
+        "for" => Some(TokenType::For),
+        "if" => Some(TokenType::If),
+        "or" => Some(TokenType::Or),
+        "while" => Some(TokenType::While),
+
         _ => None,
     }
 }

src/lib.rs

@@ -1,3 +1,103 @@
+use crate::chunk::Chunk;
+use crate::opcode::{OP_ADD,OP_SUBTRACT, OP_MULTIPLY, OP_DIVIDE, OP_CONSTANT, OP_NEGATE, OP_RETURN};
+use crate::value::Value;
+
+pub mod chunk;
 mod keywords;
-mod scanner;
+pub mod opcode;
+pub mod scanner;
 mod tokens;
+mod value;
+pub mod compiler;
+
+pub fn interpret(chunk: Chunk) -> Result {
+    let mut vm = Vm {
+        chunk,
+        ip: 0,
+        stack: vec![],
+    };
+    vm.run()
+}
+
+pub struct Vm {
+    chunk: Chunk,
+    ip: usize,
+    stack: Vec<Value>,
+}
+
+impl Vm {
+    fn run(&mut self) -> Result {
+        loop {
+            print!("[");
+            for value in self.stack.iter() {
+                print!("{:?} ", value);
+            }
+            println!("]");
+            let opcode = self.chunk.code[self.ip];
+            self.ip += 1;
+            match opcode {
+                OP_CONSTANT => {
+                    let value = &self.chunk.constants[self.chunk.code[self.ip] as usize];
+                    self.ip += 1;
+                    self.push(value.clone());
+                }
+                OP_ADD => {
+                    let a = self.pop();
+                    let b = self.pop();
+                    self.push(binary_op(a, b, |a, b| a + b))
+                }
+                OP_SUBTRACT => {
+                    let a = self.pop();
+                    let b = self.pop();
+                    self.push(binary_op(a, b, |a, b| a - b))
+                }
+                OP_MULTIPLY => {
+                    let a = self.pop();
+                    let b = self.pop();
+                    self.push(binary_op(a, b, |a, b| a * b))
+                }
+                OP_DIVIDE => {
+                    let a = self.pop();
+                    let b = self.pop();
+                    self.push(binary_op(a, b, |a, b| a / b))
+                }
+                OP_NEGATE => {
+                    let value = self.pop();
+                    self.push(-value);
+                }
+                // OP_RETURN => {
+                //     println!("{:?}", self.pop());
+                //     return Result::Ok;
+                // }
+                OP_RETURN => {
+                    println!("return {:?}", self.pop());
+                    return Result::Ok;
+                }
+                _ => {}
+            }
+        }
+    }
+
+    fn reset_stack(&mut self) {
+        self.stack.clear();
+    }
+
+    fn push(&mut self, value: Value) {
+        self.stack.push(value);
+    }
+
+    fn pop(&mut self) -> Value {
+        self.stack.pop().unwrap() //?
+    }
+}
+
+fn binary_op(a: Value, b: Value, op: impl Fn(Value, Value) -> Value) -> Value {
+    op(a, b)
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Result {
+    Ok,
+    CompileError,
+    Error,
+}

src/main.rs

@@ -1,81 +1,26 @@
-use std::sync::Arc;
+// use crudlang::chunk::Chunk;
+use crudlang::compiler::compile;
+use crudlang::interpret;
+// use crudlang::scanner::scan;
 
-use axum::{
-    Json, Router,
-    extract::{Path, State},
-    http::StatusCode,
-    routing::get,
-};
-use chrono::{DateTime, Days, Timelike, Utc};
-use dotenv::dotenv;
-use serde::{Deserialize, Serialize};
-use tokio_postgres::Client;
+fn main() -> anyhow::Result<()> {
+    // let mut chunk = Chunk::new("main");
+    // let constant = chunk.add_constant(1.2);
+    // chunk.add(crudlang::opcode::OP_CONSTANT, 123);
+    // chunk.add(constant as u16, 123);
+    // chunk.add(crudlang::opcode::OP_NEGATE, 123);
+    //
+    // let constant = chunk.add_constant(3.4);
+    // chunk.add(crudlang::opcode::OP_CONSTANT, 123);
+    // chunk.add(constant as u16, 123);
+    // chunk.add(crudlang::opcode::OP_ADD, 123);
+    //
+    // chunk.add(crudlang::opcode::OP_RETURN, 123);
+    let chunk = compile("1 + 2")?;
+    chunk.disassemble();
 
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    dotenv().ok();
-    let (client, connection) = tokio_postgres::connect(
-        "host=localhost user=postgres password=boompje dbname=postgres",
-        tokio_postgres::NoTls,
-    )
-    .await?;
-
-    // Spawn connection handler
-    tokio::spawn(async move {
-        if let Err(e) = connection.await {
-            eprintln!("connection error: {}", e);
-        }
-    });
-
-    let state = AppState {
-        db: Arc::new(client),
-    };
-
-    let app = Router::new()
-        .route("/api/customers/{id}", get(get_customer))
-        .with_state(state);
-
-    // run our app with hyper, listening globally on port 3000
-    let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap();
-    axum::serve(listener, app).await.unwrap();
+    let result = interpret(chunk);
+    println!("{:?}",result);
     Ok(())
 }
 
-#[derive(Clone)]
-struct AppState {
-    db: Arc<Client>,
-}
-
-async fn get_customer(
-    Path(id): Path<i32>,
-    State(state): State<AppState>,
-) -> Result<Json<Customer>, StatusCode> {
-    let rows = state
-        .db
-        .query(
-            "SELECT id, first_name, last_name FROM customers WHERE id = $1",
-            &[&id],
-        )
-        .await
-        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
-
-    if rows.is_empty() {
-        return Err(StatusCode::NOT_FOUND);
-    }
-
-    let row = &rows[0];
-    let user = Customer {
-        id: row.get(0),
-        first_name: row.get(1),
-        last_name: row.get(2),
-    };
-
-    Ok(Json(user))
-}
-
-#[derive(Serialize, Deserialize)]
-struct Customer {
-    id: i32,
-    first_name: String,
-    last_name: String,
-}

src/opcode.rs

@@ -0,0 +1,8 @@
+pub const OP_CONSTANT: u16 = 1;
+pub const OP_ADD: u16 = 2;
+pub const OP_SUBTRACT: u16 = 3;
+pub const OP_MULTIPLY: u16 = 4;
+pub const OP_DIVIDE: u16 = 5;
+pub const OP_NEGATE: u16 = 6;
+pub const OP_PRINT: u16 = 7;
+pub const OP_RETURN: u16 = 8;

src/reference_main.rs

@@ -0,0 +1,81 @@
+use std::sync::Arc;
+
+use axum::{
+    Json, Router,
+    extract::{Path, State},
+    http::StatusCode,
+    routing::get,
+};
+use chrono::{DateTime, Days, Timelike, Utc};
+use dotenv::dotenv;
+use serde::{Deserialize, Serialize};
+use tokio_postgres::Client;
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    dotenv().ok();
+    let (client, connection) = tokio_postgres::connect(
+        "host=localhost user=postgres password=boompje dbname=postgres",
+        tokio_postgres::NoTls,
+    )
+    .await?;
+
+    // Spawn connection handler
+    tokio::spawn(async move {
+        if let Err(e) = connection.await {
+            eprintln!("connection error: {}", e);
+        }
+    });
+
+    let state = AppState {
+        db: Arc::new(client),
+    };
+
+    let app = Router::new()
+        .route("/api/customers/{id}", get(get_customer))
+        .with_state(state);
+
+    // run our app with hyper, listening globally on port 3000
+    let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap();
+    axum::serve(listener, app).await.unwrap();
+    Ok(())
+}
+
+#[derive(Clone)]
+struct AppState {
+    db: Arc<Client>,
+}
+
+async fn get_customer(
+    Path(id): Path<i32>,
+    State(state): State<AppState>,
+) -> Result<Json<Customer>, StatusCode> {
+    let rows = state
+        .db
+        .query(
+            "SELECT id, first_name, last_name FROM customers WHERE id = $1",
+            &[&id],
+        )
+        .await
+        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
+
+    if rows.is_empty() {
+        return Err(StatusCode::NOT_FOUND);
+    }
+
+    let row = &rows[0];
+    let user = Customer {
+        id: row.get(0),
+        first_name: row.get(1),
+        last_name: row.get(2),
+    };
+
+    Ok(Json(user))
+}
+
+#[derive(Serialize, Deserialize)]
+struct Customer {
+    id: i32,
+    first_name: String,
+    last_name: String,
+}

src/scanner.rs

@@ -6,13 +6,14 @@ use crate::{
     },
 };
 
-fn scan(source: &str) -> Vec<Token> {
+pub fn scan(source: &str) -> Vec<Token> {
     let scanner = Scanner {
         chars: source.chars().collect(),
         current: 0,
         start: 0,
         line: 0,
         tokens: vec![],
+        new_line: true,
     };
     scanner.scan()
 }
@@ -23,17 +24,26 @@ impl Scanner {
             self.start = self.current;
             self.scan_token();
         }
-
+        self.add_token(TokenType::Eof);
         self.tokens
     }
 
     fn scan_token(&mut self) {
         let c = self.advance();
+        if self.new_line && (c == ' ' || c == '\t') {
+            self.add_token(TokenType::Indent);
+            self.new_line = false;
+        } else {
+            if c != '\n' {
+                self.new_line = false;
+            }
             match c {
                 '(' => self.add_token(TokenType::LeftParen),
                 ')' => self.add_token(TokenType::RightParen),
                 '{' => self.add_token(TokenType::LeftBrace),
                 '}' => self.add_token(TokenType::RightBrace),
+                '[' => self.add_token(TokenType::LeftBracket),
+                ']' => self.add_token(TokenType::RightBracket),
                 ',' => self.add_token(TokenType::Comma),
                 '.' => self.add_token(TokenType::Dot),
                 '-' => self.add_token(TokenType::Minus),
@@ -83,9 +93,10 @@ impl Scanner {
                     }
                 }
                 '"' => self.string(),
-            ' ' | '\t' | '\r' => {}
+                '\r' | '\t' | ' ' => {}
                 '\n' => {
                     self.line += 1;
+                    self.new_line = true;
                 }
                 _ => {
                     if is_digit(c) {
@@ -98,6 +109,7 @@ impl Scanner {
                 }
             }
         }
+    }
 
     fn identifier(&mut self) {
         while is_alphanumeric(self.peek()) {
@@ -146,8 +158,12 @@ impl Scanner {
     }
 
     fn peek(&self) -> char {
+        if self.current>=self.chars.len(){
+            '\0'
+        } else {
             self.chars[self.current]
         }
+    }
 
     fn peek_next(&self) -> char {
         self.chars[self.current + 1]
@@ -188,6 +204,7 @@ struct Scanner {
     start: usize,
     tokens: Vec<Token>,
     line: usize,
+    new_line: bool,
 }
 
 fn is_digit(c: char) -> bool {
@@ -199,7 +216,7 @@ fn is_alphanumeric(c: char) -> bool {
 }
 
 fn is_alpha(c: char) -> bool {
-    (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+    (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '$'
 }
 
 #[cfg(test)]
@@ -210,13 +227,14 @@ mod test {
     #[test]
     fn  test() {
         let tokens = scan(
-            r#"
-            ///gets the customer
-            fn get(id: u32) -> Customer:
-                service.get(id)
-            "#,
+            r#"struct Customer:
+                        id: u32,
+                        first_name: string,
+                        last_name: string,
+                        date_fetched: date,"#,
         );
         let tokenstring = format!("{:?}", tokens);
-        assert_eq!(tokenstring,r#"[Token { tokentype: Fn, lexeme: "fn", line: 2 }, Token { tokentype: Identifier, lexeme: "get", line: 2 }, Token { tokentype: LeftParen, lexeme: "", line: 2 }, Token { tokentype: Identifier, lexeme: "id", line: 2 }, Token { tokentype: Colon, lexeme: "", line: 2 }, Token { tokentype: Identifier, lexeme: "u32", line: 2 }, Token { tokentype: RightParen, lexeme: "", line: 2 }, Token { tokentype: Minus, lexeme: "", line: 2 }, Token { tokentype: Greater, lexeme: "", line: 2 }, Token { tokentype: Identifier, lexeme: "Customer", line: 2 }, Token { tokentype: Colon, lexeme: "", line: 2 }, Token { tokentype: Identifier, lexeme: "service", line: 3 }, Token { tokentype: Dot, lexeme: "", line: 3 }, Token { tokentype: Identifier, lexeme: "get", line: 3 }, Token { tokentype: LeftParen, lexeme: "", line: 3 }, Token { tokentype: Identifier, lexeme: "id", line: 3 }, Token { tokentype: RightParen, lexeme: "", line: 3 }]"#)
+        println!("{}", tokenstring);
+        // assert_eq!(tokenstring,r#"[Token { tokentype: Fn, lexeme: "fn", line: 2 }, Token { tokentype: Identifier, lexeme: "get", line: 2 }, Token { tokentype: LeftParen, lexeme: "", line: 2 }, Token { tokentype: Identifier, lexeme: "id", line: 2 }, Token { tokentype: Colon, lexeme: "", line: 2 }, Token { tokentype: Identifier, lexeme: "u32", line: 2 }, Token { tokentype: RightParen, lexeme: "", line: 2 }, Token { tokentype: Minus, lexeme: "", line: 2 }, Token { tokentype: Greater, lexeme: "", line: 2 }, Token { tokentype: Identifier, lexeme: "Customer", line: 2 }, Token { tokentype: Colon, lexeme: "", line: 2 }, Token { tokentype: Identifier, lexeme: "service", line: 3 }, Token { tokentype: Dot, lexeme: "", line: 3 }, Token { tokentype: Identifier, lexeme: "get", line: 3 }, Token { tokentype: LeftParen, lexeme: "", line: 3 }, Token { tokentype: Identifier, lexeme: "id", line: 3 }, Token { tokentype: RightParen, lexeme: "", line: 3 }]"#)
     }
 }

src/tokens.rs

@@ -1,8 +1,8 @@
 #[derive(Debug)]
-pub(crate) struct Token {
-    tokentype: TokenType,
-    lexeme: String,
-    line: usize,
+pub struct Token {
+    pub tokentype: TokenType,
+    pub lexeme: String,
+    pub line: usize,
 }
 
 impl Token {
@@ -20,8 +20,9 @@ enum Value {
     None,
 }
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq, Clone, Copy, Hash)]
 pub(crate) enum TokenType {
+    Error,
     LeftParen,
     RightParen,
     LeftBrace,
@@ -48,6 +49,28 @@ pub(crate) enum TokenType {
     Identifier,
     String,
     Number,
+    And,
     Fn,
     Struct,
+    Else,
+    False,
+    True,
+    Null,
+    If,
+    Or,
+    While,
+    For,
+    Return,
+    Print,
+    Eof,
+    U32Type,
+    U64Type,
+    I32Type,
+    I64Type,
+    DateType,
+    StringType,
+}
+
+impl Eq for TokenType {
+
 }

src/value.rs

@@ -0,0 +1,185 @@
+use chrono::Utc;
+use std::collections::HashMap;
+use std::ops::{Add, Div, Mul, Neg, Sub};
+
+#[derive(Debug, Clone)]
+pub enum Value {
+    U32(u32),
+    I32(i32),
+    U64(u64),
+    I64(i64),
+    F32(f32),
+    F64(f64),
+    String(String),
+    Char(char),
+    Bool(bool),
+    Date(Utc),
+    Enum,
+    Struct,
+    List(Vec<Value>),
+    Map(HashMap<Value, Value>),
+}
+
+impl Into<Value> for i32 {
+    fn into(self) -> Value {
+        Value::I32(self)
+    }
+}
+
+impl Into<Value> for i64 {
+    fn into(self) -> Value {
+        Value::I64(self)
+    }
+}
+
+impl Into<Value> for u32 {
+    fn into(self) -> Value {
+        Value::U32(self)
+    }
+}
+
+impl Into<Value> for u64 {
+    fn into(self) -> Value {
+        Value::U64(self)
+    }
+}
+
+impl Into<Value> for f32 {
+    fn into(self) -> Value {
+        Value::F32(self)
+    }
+}
+
+impl Into<Value> for f64 {
+    fn into(self) -> Value {
+        Value::F64(self)
+    }
+}
+
+impl Into<Value> for &str {
+    fn into(self) -> Value {
+        Value::String(self.to_string())
+    }
+}
+impl Into<Value> for char {
+    fn into(self) -> Value {
+        Value::Char(self)
+    }
+}
+
+impl Into<Value> for bool {
+    fn into(self) -> Value {
+        Value::Bool(self)
+    }
+}
+
+impl Into<Value> for Utc {
+    fn into(self) -> Value {
+        Value::Date(self)
+    }
+}
+
+impl Into<Value> for Vec<Value> {
+    fn into(self) -> Value {
+        Value::List(self)
+    }
+}
+
+impl Into<Value> for HashMap<Value, Value> {
+    fn into(self) -> Value {
+        Value::Map(self)
+    }
+}
+
+impl Neg for Value {
+    type Output = Value;
+
+    fn neg(self) -> Self::Output {
+        match self {
+            Value::I32(i) => Value::I32(-i),
+            Value::I64(i) => Value::I64(-i),
+            Value::F32(i) => Value::F32(-i),
+            Value::F64(i) => Value::F64(-i),
+            _ => panic!("Cannot negate {:?}", self),
+        }
+    }
+}
+
+impl Add<Value> for Value {
+    type Output = Value;
+
+    fn add(self, rhs: Value) -> Self::Output {
+        match (self, rhs) {
+            (Value::I32(a), Value::I32(b)) => Value::I32(a + b),
+            (Value::I64(a), Value::I64(b)) => Value::I64(a + b),
+            (Value::U32(a), Value::U32(b)) => Value::U32(a + b),
+            (Value::U64(a), Value::U64(b)) => Value::U64(a + b),
+            (Value::F32(a), Value::F32(b)) => Value::F32(a + b),
+            (Value::F64(a), Value::F64(b)) => Value::F64(a + b),
+            (Value::String(s), Value::I32(i)) => Value::String(format!("{}{}", s, i)),
+            (Value::String(s), Value::I64(i)) => Value::String(format!("{}{}", s, i)),
+            (Value::String(s), Value::U32(u)) => Value::String(format!("{}{}", s, u)),
+            (Value::String(s), Value::U64(u)) => Value::String(format!("{}{}", s, u)),
+            (Value::String(s), Value::F32(f)) => Value::String(format!("{}{}", s, f)),
+            (Value::String(s), Value::F64(f)) => Value::String(format!("{}{}", s, f)),
+            (Value::String(s), Value::Bool(b)) => Value::String(format!("{}{}", s, b)),
+            (Value::String(s), Value::Char(c)) => Value::String(format!("{}{}", s, c)),
+            (Value::String(s1), Value::String(s2)) => Value::String(format!("{}{}", s1, s2)),
+            (Value::String(s1), Value::List(l)) => Value::String(format!("{}{:?}", s1, l)),
+            (Value::String(s1), Value::Map(m)) => Value::String(format!("{}{:?}", s1, m)),
+            //enum?
+            _ => panic!("Cannot add"),
+        }
+    }
+}
+
+impl Sub<Value> for Value {
+    type Output = Value;
+
+    fn sub(self, rhs: Value) -> Self::Output {
+        match (self, rhs) {
+            (Value::I32(a), Value::I32(b)) => Value::I32(a - b),
+            (Value::I64(a), Value::I64(b)) => Value::I64(a - b),
+            (Value::U32(a), Value::U32(b)) => Value::U32(a - b),
+            (Value::U64(a), Value::U64(b)) => Value::U64(a - b),
+            (Value::F32(a), Value::F32(b)) => Value::F32(a - b),
+            (Value::F64(a), Value::F64(b)) => Value::F64(a - b),
+            //enum?
+            _ => panic!("Cannot subtract"),
+        }
+    }
+}
+
+impl Mul<Value> for Value {
+    type Output = Value;
+
+    fn mul(self, rhs: Value) -> Self::Output {
+        match (self, rhs) {
+            (Value::I32(a), Value::I32(b)) => Value::I32(a * b),
+            (Value::I64(a), Value::I64(b)) => Value::I64(a * b),
+            (Value::U32(a), Value::U32(b)) => Value::U32(a * b),
+            (Value::U64(a), Value::U64(b)) => Value::U64(a * b),
+            (Value::F32(a), Value::F32(b)) => Value::F32(a * b),
+            (Value::F64(a), Value::F64(b)) => Value::F64(a * b),
+            //enum?
+            _ => panic!("Cannot multiply"),
+        }
+    }
+}
+
+impl Div<Value> for Value {
+    type Output = Value;
+
+    fn div(self, rhs: Value) -> Self::Output {
+        match (self, rhs) {
+            (Value::I32(a), Value::I32(b)) => Value::I32(a / b),
+            (Value::I64(a), Value::I64(b)) => Value::I64(a / b),
+            (Value::U32(a), Value::U32(b)) => Value::U32(a / b),
+            (Value::U64(a), Value::U64(b)) => Value::U64(a / b),
+            (Value::F32(a), Value::F32(b)) => Value::F32(a / b),
+            (Value::F64(a), Value::F64(b)) => Value::F64(a / b),
+            //enum?
+            _ => panic!("Cannot divide"),
+        }
+    }
+}