tests

Cargo.lock

@@ -124,9 +124,6 @@ name = "bumpalo"
 version = "3.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
-dependencies = [
- "serde",
-]
 
 [[package]]
 name = "byteorder"
@@ -200,7 +197,6 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "axum",
- "bumpalo",
  "chrono",
  "dotenv",
  "log",

Cargo.toml

@@ -20,5 +20,4 @@ tracing-subscriber = "0.3.20"
 anyhow = "1.0"
 tower-livereload = "0.9.6"
 log = "0.4.28"
-bumpalo = { version = "3.19", features = ["collections", "boxed", "serde"] }
 walkdir = "2.5.0"

README.md

@@ -100,9 +100,9 @@ And I cherry picked things I like, mostly from rust and python.
 ```
 let a = 42
 ```
-* declares a variable of type i64
+* declares a variable of type i64 (signed 64 bit integer)
 
-or explictly as u32
+or explictly as u32 (unsigned 32 bit integer)
 ```
 let a:u32 = 42
 ```

src/ast_compiler.rs

@@ -1,11 +1,7 @@
-use crate::tokens::TokenType::{
+use crate::tokens::TokenType::{Bang, Bool, Colon, Date, Eol, Equal, F32, F64, False, FloatingPoint, Fn, Greater, GreaterEqual, GreaterGreater, I32, I64, Identifier, If, Indent, Integer, LeftBracket, LeftParen, Less, LessEqual, LessLess, Let, ListType, MapType, Minus, Object, Plus, Print, RightBracket, RightParen, SignedInteger, SingleRightArrow, Slash, Star, StringType, True, U32, U64, UnsignedInteger, Char};
-    Bang, Bool, Char, Colon, Date, Eol, Equal, F32, F64, False, FloatingPoint, Fn, Greater,
-    GreaterEqual, GreaterGreater, I32, I64, Identifier, If, Indent, Integer, LeftBracket,
-    LeftParen, Less, LessEqual, LessLess, Let, ListType, MapType, Minus, Object, Plus, Print,
-    RightBracket, RightParen, SingleRightArrow, Slash, Star, StringType, True, U32, U64,
-};
 use crate::tokens::{Token, TokenType};
 use crate::value::Value;
+use anyhow::anyhow;
 use log::debug;
 use std::collections::HashMap;
 
@@ -15,7 +11,7 @@ pub fn compile(tokens: Vec<Token>) -> anyhow::Result<Vec<Statement>> {
 }
 
 #[derive(Debug, Clone)]
-pub(crate) struct Function {
+pub struct Function {
     pub(crate) name: Token,
     pub(crate) parameters: Vec<Parameter>,
     pub(crate) return_type: TokenType,
@@ -53,7 +49,7 @@ impl AstCompiler {
         self.compile(expected_indent)
     }
 
-    fn compile(&mut self, expected_indent: usize) -> anyhow::Result<Vec<Statement>>{
+    fn compile(&mut self, expected_indent: usize) -> anyhow::Result<Vec<Statement>> {
         if !self.had_error {
             let mut statements = vec![];
             while !self.is_at_end() {
@@ -201,11 +197,13 @@ impl AstCompiler {
             let var_type = match calculate_type(declared_type, inferred_type) {
                 Ok(var_type) => var_type,
                 Err(e) => {
-                    println!("error at line {}", name_token.line);
                     self.had_error = true;
-                    return Err(e);
+                    return Err(anyhow!("error at line {}: {}", name_token.line, e));
                 }
             };
+            // match var_type{
+            //     U32 => U32()
+            // }
             self.vars.push(Expression::Variable {
                 name: name_token.lexeme.to_string(),
                 var_type,
@@ -246,7 +244,7 @@ impl AstCompiler {
     }
 
     fn or(&mut self) -> anyhow::Result<Expression> {
-        let mut expr = self.and()?;
+        let expr = self.and()?;
         self.binary(vec![TokenType::LogicalOr], expr)
     }
 
@@ -281,7 +279,7 @@ impl AstCompiler {
     }
 
     fn bitshift(&mut self) -> anyhow::Result<Expression> {
-        let mut expr = self.term()?;
+        let expr = self.term()?;
         self.binary(vec![GreaterGreater, LessLess], expr)
     }
 
@@ -361,6 +359,12 @@ impl AstCompiler {
                 literaltype: StringType,
                 value: Value::String(self.previous().lexeme.to_string()),
             }
+        } else if self.match_token(vec![Char]) {
+            Expression::Literal {
+                line: self.peek().line,
+                literaltype: Char,
+                value: Value::Char(self.previous().lexeme.chars().next().unwrap()),
+            }
         } else if self.match_token(vec![LeftParen]) {
             let expr = self.expression()?;
             self.consume(RightParen, "Expect ')' after expression.")?;
@@ -418,7 +422,6 @@ impl AstCompiler {
     }
 
     fn function_call(&mut self, name: String) -> anyhow::Result<Expression> {
-        println!("function call {}", name);
         let function_name = self.functions.get(&name).unwrap().name.lexeme.clone();
         let function = self.functions.get(&function_name).unwrap().clone();
 
@@ -509,20 +512,20 @@ fn calculate_type(
     declared_type: Option<TokenType>,
     inferred_type: TokenType,
 ) -> anyhow::Result<TokenType> {
-    println!(
-        "declared type {:?} inferred type: {:?}",
-        declared_type, inferred_type
-    );
     Ok(if let Some(declared_type) = declared_type {
         if declared_type != inferred_type {
             match (declared_type, inferred_type) {
-                (I32, I64) => I32,
+                (I32, I64) => I32, //need this?
+                (I32, Integer) => I32,
                 (U32, U64) => U32,
+                (U32, Integer) => U32,
                 (F32, F64) => F32,
+                (F32, FloatingPoint) => F32,
                 (F64, I64) => F64,
+                (F64, FloatingPoint) => F64,
                 (U64, I64) => U64,
                 (U64, I32) => U64,
-                (StringType, _) => StringType, // meh, this all needs rigorous testing
+                (StringType, _) => StringType, // meh, this all needs rigorous testing. Update: this is in progress
                 _ => {
                     return Err(anyhow::anyhow!(
                         "Incompatible types. Expected {}, found {}",
@@ -697,7 +700,16 @@ impl Expression {
             Self::Grouping { expression, .. } => expression.infer_type(),
             Self::Literal { literaltype, .. } => literaltype.clone(),
             Self::List { literaltype, .. } => literaltype.clone(),
-            Self::Unary { right, .. } => right.infer_type(),
+            Self::Unary {
+                right, operator, ..
+            } => {
+                let literal_type = right.infer_type();
+                if literal_type == Integer && operator.token_type == Minus {
+                    SignedInteger
+                } else {
+                    UnsignedInteger
+                }
+            }
             Self::Variable { var_type, .. } => var_type.clone(),
             Self::FunctionCall { return_type, .. } => return_type.clone(),
         }

src/bytecode_compiler.rs

@@ -9,7 +9,6 @@ use crate::vm::{
     OP_GREATER_EQUAL, OP_LESS, OP_LESS_EQUAL, OP_MULTIPLY, OP_NEGATE, OP_NOT, OP_OR, OP_PRINT,
     OP_RETURN, OP_SHL, OP_SHR, OP_SUBTRACT,
 };
-use anyhow::anyhow;
 use std::collections::HashMap;
 
 pub fn compile(
@@ -47,7 +46,7 @@ pub(crate) fn compile_name(
 
 struct Compiler {
     chunk: Chunk,
-    had_error: bool,
+    _had_error: bool,
     current_line: usize,
     vars: HashMap<String, usize>,
 }
@@ -56,7 +55,7 @@ impl Compiler {
     fn new(name: &str) -> Self {
         Self {
             chunk: Chunk::new(name),
-            had_error: false,
+            _had_error: false,
             current_line: 0,
             vars: HashMap::new(),
         }
@@ -142,7 +141,6 @@ impl Compiler {
                 } else {
                     name.clone()
                 };
-                println!("call {}", name);
                 let name_index = self
                     .chunk
                     .find_constant(&name)

src/compiler_tests.rs

@@ -0,0 +1,94 @@
+#[cfg(test)]
+mod tests {
+    use crate::compile;
+    use crate::scanner::scan;
+
+    #[test]
+    fn literal_int() {
+        assert!(compile("1").is_ok());
+    }
+
+    #[test]
+    fn literal_float() {
+        assert!(compile("2.1").is_ok());
+    }
+
+    #[test]
+    fn literal_float_scientific() {
+        assert!(compile("2.1e5").is_ok());
+    }
+
+    #[test]
+    fn literal_string() {
+        assert!(compile(r#""a""#).is_ok());
+    }
+
+    #[test]
+    fn literal_list() {
+        assert!(compile(r#"["abc","def"]"#).is_ok());
+    }
+
+    #[test]
+    fn let_infer_type() {
+        assert!(compile(r#"let a=1"#).is_ok());
+    }
+
+    #[test]
+    fn let_u32() {
+        assert!(compile(r#"let a:u32=1"#).is_ok());
+    }
+
+    #[test]
+    fn let_char() {
+        assert!(scan(r#"let a:char='a'"#).is_ok());
+    }
+
+    #[test]
+    fn let_u32_invalid_value_negative() {
+        let r = compile("let a:u32=-1");
+        assert!(r.is_err());
+        if let Err(e) = &r {
+            assert_eq!(
+                e.to_string(),
+                "error at line 1: Incompatible types. Expected u32, found i32/64"
+            );
+        }
+    }
+
+    #[test]
+    fn let_u64_invalid_value_negative() {
+        let r = compile("let a:u64=-1");
+        assert!(r.is_err());
+        if let Err(e) = &r {
+            assert_eq!(
+                e.to_string(),
+                "error at line 1: Incompatible types. Expected u64, found i32/64"
+            );
+        }
+    }
+
+    #[test]
+    fn let_u64_invalid_value_string() {
+        let r = compile(r#"let a:u64="not ok""#);
+        assert!(r.is_err());
+        if let Err(e) = &r {
+            assert_eq!(
+                e.to_string(),
+                "error at line 1: Incompatible types. Expected u64, found string"
+            );
+        }
+    }
+
+    #[test]
+    fn call_fn_with_args_returns_value() {
+        assert!(
+            compile(
+                r#"
+fn hello(name: string) -> string:
+    "Hello " + name
+hello("world")"#
+            )
+            .is_ok()
+        );
+    }
+}

src/lib.rs

@@ -1,8 +1,20 @@
-pub mod scanner;
+use std::collections::HashMap;
+use crate::scanner::scan;
+
 pub mod ast_compiler;
 pub mod bytecode_compiler;
-pub mod vm;
 pub mod chunk;
 mod keywords;
+pub mod scanner;
+mod compiler_tests;
 mod tokens;
 mod value;
+pub mod vm;
+
+pub fn compile(src: &str) -> anyhow::Result<chunk::Chunk> {
+    let tokens = scan(src)?;
+    let mut registry = HashMap::new();
+    let ast= ast_compiler::compile(tokens)?;
+    let bytecode = bytecode_compiler::compile("", &ast, &mut registry)?;
+    Ok(bytecode)
+}

src/main.rs

@@ -24,7 +24,7 @@ async fn main() -> anyhow::Result<()> {
         if path.is_file() && path.ends_with("web.crud") {
             print!("compiling {:?}: ", path);
             let source = fs::read_to_string(path)?;
-            let tokens = scan(&source);
+            let tokens = scan(&source)?;
             match ast_compiler::compile(tokens) {
                 Ok(statements) => {
                     let path = path
@@ -53,7 +53,10 @@ async fn main() -> anyhow::Result<()> {
                 registry: registry.clone(),
             });
             println!("adding {}", path);
-            app = app.route(&format!("/{}",path.replace("/web", "")), get(handle_get).with_state(state.clone()));
+            app = app.route(
+                &format!("/{}", path.replace("/web", "")),
+                get(handle_get).with_state(state.clone()),
+            );
         }
         let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await?;
         println!("listening on {}", listener.local_addr()?);
@@ -77,33 +80,3 @@ async fn handle_get(State(state): State<Arc<AppState>>) -> Result<Json<String>,
     ))
 }
 
-//
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[tokio::test]
-    async fn test_compile() -> anyhow::Result<()> {
-        let tokens = scan(
-            r#"
-fn hello(name: string) -> string:
-    "Hello "+name
-hello("sander")"#,
-        );
-        let mut registry = HashMap::new();
-        match ast_compiler::compile(tokens) {
-            Ok(statements) => {
-                println!("{:?}", statements);
-                let chunk = compile("", &statements, &mut registry)?;
-                chunk.disassemble();
-                // println!("{}", interpret(&chunk).await?);
-            }
-            Err(e) => {
-                println!("{}", e)
-            }
-        }
-        println!("{:?}", registry);
-        Ok(())
-    }
-}

src/scanner.rs

@@ -1,3 +1,4 @@
+use anyhow::anyhow;
 use crate::tokens::TokenType::{BitXor, FloatingPoint, Integer};
 use crate::{
     keywords,
@@ -7,7 +8,7 @@ use crate::{
     },
 };
 
-pub fn scan(source: &str) -> Vec<Token> {
+pub fn scan(source: &str) -> anyhow::Result<Vec<Token>> {
     let scanner = Scanner {
         chars: source.chars().collect(),
         current: 0,
@@ -20,17 +21,17 @@ pub fn scan(source: &str) -> Vec<Token> {
 }
 
 impl Scanner {
-    fn scan(mut self) -> Vec<Token> {
+    fn scan(mut self) -> anyhow::Result<Vec<Token>> {
         while !self.is_at_end() {
             self.start = self.current;
-            self.scan_token();
+            self.scan_token()?;
         }
         self.add_token(TokenType::Eol);
         self.add_token(TokenType::Eof);
-        self.tokens
+        Ok(self.tokens)
     }
 
-    fn scan_token(&mut self) {
+    fn scan_token(&mut self) -> anyhow::Result<()>{
         let c = self.advance();
         if self.new_line && (c == ' ' || c == '\t') {
             self.add_token(TokenType::Indent);
@@ -106,7 +107,8 @@ impl Scanner {
                         self.add_token(TokenType::Slash);
                     }
                 }
-                '"' => self.string(),
+                '\'' => self.char()?,
+                '"' => self.string()?,
                 '\r' | '\t' | ' ' => {}
                 '\n' => {
                     self.line += 1;
@@ -136,11 +138,12 @@ impl Scanner {
                     } else if is_alpha(c) {
                         self.identifier();
                     } else {
-                        println!("Unexpected identifier at line {}", self.line);
+                        return Err(anyhow!("Unexpected identifier at line {}", self.line));
                     }
                 }
             }
         }
+        Ok(())
     }
 
     fn identifier(&mut self) {
@@ -154,7 +157,7 @@ impl Scanner {
     }
 
     fn number(&mut self) {
-        while is_digit(self.peek()) {
+        while is_digit(self.peek() ) {
             self.advance();
         }
         let mut has_dot = false;
@@ -163,14 +166,35 @@ impl Scanner {
             self.advance();
         }
 
-        while is_digit(self.peek()) {
+        while is_digit_or_scientific(self.peek()) {
             self.advance();
         }
         let value: String = self.chars[self.start..self.current].iter().collect();
         self.add_token_with_value(if has_dot { FloatingPoint } else { Integer }, value);
     }
 
-    fn string(&mut self) {
+    fn char(&mut self) -> anyhow::Result<()>{
+        while self.peek() != '\'' && !self.is_at_end() {
+            self.advance();
+        }
+
+        if self.is_at_end() {
+            return Err(anyhow!("Unterminated char at {}", self.line))
+        }
+
+        self.advance();
+
+        let value: String = self.chars[self.start + 1..self.current - 1]
+            .iter()
+            .collect();
+        if value.len() != 1 {
+            return Err(anyhow!("Illegal char length for {} at line {}", value, self.line))
+        }
+        self.add_token_with_value(TokenType::Char, value);
+        Ok(())
+    }
+
+    fn string(&mut self) -> anyhow::Result<()>{
         while self.peek() != '"' && !self.is_at_end() {
             if self.peek() == '\n' {
                 self.line += 1;
@@ -179,7 +203,7 @@ impl Scanner {
         }
 
         if self.is_at_end() {
-            println!("Unterminated string at {}", self.line)
+            return Err(anyhow!("Unterminated string at {}", self.line))
         }
 
         self.advance();
@@ -188,6 +212,7 @@ impl Scanner {
             .iter()
             .collect();
         self.add_token_with_value(TokenType::StringType, value);
+        Ok(())
     }
 
     fn peek(&self) -> char {
@@ -244,6 +269,10 @@ fn is_digit(c: char) -> bool {
     c >= '0' && c <= '9'
 }
 
+fn is_digit_or_scientific(c: char) -> bool {
+    is_digit(c) || c=='e' || c=='E'
+}
+
 fn is_alphanumeric(c: char) -> bool {
     is_alpha(c) || is_digit(c)
 }

src/tokens.rs

@@ -17,11 +17,6 @@ impl Token {
     }
 }
 
-#[derive(Debug)]
-enum Value {
-    None,
-}
-
 #[derive(Debug, PartialEq, Clone, Copy, Hash)]
 pub enum TokenType {
     Bang,
@@ -55,7 +50,9 @@ pub enum TokenType {
     Identifier,
     If,
     Indent,
-    Integer, //undetermined integer type
+    Integer,
+    SignedInteger,
+    UnsignedInteger,
     LeftBrace,
     LeftBracket,
     LeftParen,
@@ -69,7 +66,7 @@ pub enum TokenType {
     LogicalOr,
     Minus,
     Not,
-    FloatingPoint, //undetermined float type
+    FloatingPoint,
     Object,
     Plus,
     Print,
@@ -157,6 +154,8 @@ impl fmt::Display for TokenType {
             TokenType::True => write!(f, "true"),
             TokenType::Void => write!(f, "()"),
             TokenType::While => write!(f, "while"),
+            TokenType::SignedInteger => write!(f, "i32/64"),
+            TokenType::UnsignedInteger => write!(f, "u32/64"),
         }
     }
 }

src/value.rs

@@ -143,7 +143,7 @@ impl Display for Value {
             &Value::Enum => write!(f, "enum"),
             &Value::Struct(v) => write!(f, "{}", v),
             &Value::List(v) => write!(f, "{:?}", v),
-            &Value::Map(v) => write!(f, "map"),
+            &Value::Map(_) => write!(f, "map"),
             &Value::Error(v) => write!(f, "{}", v),
             &Value::Void => write!(f, "()"),
         }
@@ -295,7 +295,7 @@ impl Not for &Value {
     type Output = anyhow::Result<Value>;
 
     fn not(self) -> Self::Output {
-        match (self) {
+        match self {
             Value::Bool(b) => Ok(Value::Bool(!b)),
             Value::I32(i32) => Ok(Value::I32(!i32)),
             Value::I64(i64) => Ok(Value::I64(!i64)),

src/vm.rs

@@ -1,7 +1,6 @@
 use crate::chunk::Chunk;
 use crate::value::Value;
 use anyhow::anyhow;
-use bumpalo::Bump;
 use std::collections::HashMap;
 use tracing::debug;
 
@@ -175,10 +174,6 @@ impl <'a> Vm<'a> {
         chunk.constants[index].to_string() //string??
     }
 
-    fn reset_stack(&mut self) {
-        self.stack.clear();
-    }
-
     fn push(&mut self, value: Value) {
         self.stack.push(value);
     }