sander/tipi-lang
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tests

Browse source
This commit is contained in:
Shautvast 2025-10-30 13:50:28 +01:00
parent 90ad226785
commit 3118ce97b0
12 changed files with 197 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Cargo.lock generated
View file

@ -124,9 +124,6 @@ name = "bumpalo"
version = "3.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
dependencies = [
"serde",
]
[[package]]
name = "byteorder"
@ -200,7 +197,6 @@ version = "0.1.0"
dependencies = [
"anyhow",
"axum",
"bumpalo",
"chrono",
"dotenv",
"log",

1
Cargo.toml
View file

@ -20,5 +20,4 @@ tracing-subscriber = "0.3.20"
anyhow = "1.0"
tower-livereload = "0.9.6"
log = "0.4.28"
bumpalo = { version = "3.19", features = ["collections", "boxed", "serde"] }
walkdir = "2.5.0"

4
README.md
View file

@ -100,9 +100,9 @@ And I cherry picked things I like, mostly from rust and python.
```
let a = 42
```
* declares a variable of type i64
* declares a variable of type i64 (signed 64 bit integer)
or explictly as u32
or explictly as u32 (unsigned 32 bit integer)
```
let a:u32 = 42
```

50
src/ast_compiler.rs
View file

@ -1,11 +1,7 @@
use crate::tokens::TokenType::{
Bang, Bool, Char, Colon, Date, Eol, Equal, F32, F64, False, FloatingPoint, Fn, Greater,
GreaterEqual, GreaterGreater, I32, I64, Identifier, If, Indent, Integer, LeftBracket,
LeftParen, Less, LessEqual, LessLess, Let, ListType, MapType, Minus, Object, Plus, Print,
RightBracket, RightParen, SingleRightArrow, Slash, Star, StringType, True, U32, U64,
};
use crate::tokens::TokenType::{Bang, Bool, Colon, Date, Eol, Equal, F32, F64, False, FloatingPoint, Fn, Greater, GreaterEqual, GreaterGreater, I32, I64, Identifier, If, Indent, Integer, LeftBracket, LeftParen, Less, LessEqual, LessLess, Let, ListType, MapType, Minus, Object, Plus, Print, RightBracket, RightParen, SignedInteger, SingleRightArrow, Slash, Star, StringType, True, U32, U64, UnsignedInteger, Char};
use crate::tokens::{Token, TokenType};
use crate::value::Value;
use anyhow::anyhow;
use log::debug;
use std::collections::HashMap;
@ -15,7 +11,7 @@ pub fn compile(tokens: Vec<Token>) -> anyhow::Result<Vec<Statement>> {
}
#[derive(Debug, Clone)]
pub(crate) struct Function {
pub struct Function {
pub(crate) name: Token,
pub(crate) parameters: Vec<Parameter>,
pub(crate) return_type: TokenType,
@ -201,11 +197,13 @@ impl AstCompiler {
let var_type = match calculate_type(declared_type, inferred_type) {
Ok(var_type) => var_type,
Err(e) => {
println!("error at line {}", name_token.line);
self.had_error = true;
return Err(e);
return Err(anyhow!("error at line {}: {}", name_token.line, e));
}
};
// match var_type{
// U32 => U32()
// }
self.vars.push(Expression::Variable {
name: name_token.lexeme.to_string(),
var_type,
@ -246,7 +244,7 @@ impl AstCompiler {
}
fn or(&mut self) -> anyhow::Result<Expression> {
let mut expr = self.and()?;
let expr = self.and()?;
self.binary(vec![TokenType::LogicalOr], expr)
}
@ -281,7 +279,7 @@ impl AstCompiler {
}
fn bitshift(&mut self) -> anyhow::Result<Expression> {
let mut expr = self.term()?;
let expr = self.term()?;
self.binary(vec![GreaterGreater, LessLess], expr)
}
@ -361,6 +359,12 @@ impl AstCompiler {
literaltype: StringType,
value: Value::String(self.previous().lexeme.to_string()),
}
} else if self.match_token(vec![Char]) {
Expression::Literal {
line: self.peek().line,
literaltype: Char,
value: Value::Char(self.previous().lexeme.chars().next().unwrap()),
}
} else if self.match_token(vec![LeftParen]) {
let expr = self.expression()?;
self.consume(RightParen, "Expect ')' after expression.")?;
@ -418,7 +422,6 @@ impl AstCompiler {
}
fn function_call(&mut self, name: String) -> anyhow::Result<Expression> {
println!("function call {}", name);
let function_name = self.functions.get(&name).unwrap().name.lexeme.clone();
let function = self.functions.get(&function_name).unwrap().clone();
@ -509,20 +512,20 @@ fn calculate_type(
declared_type: Option<TokenType>,
inferred_type: TokenType,
) -> anyhow::Result<TokenType> {
println!(
"declared type {:?} inferred type: {:?}",
declared_type, inferred_type
);
Ok(if let Some(declared_type) = declared_type {
if declared_type != inferred_type {
match (declared_type, inferred_type) {
(I32, I64) => I32,
(I32, I64) => I32, //need this?
(I32, Integer) => I32,
(U32, U64) => U32,
(U32, Integer) => U32,
(F32, F64) => F32,
(F32, FloatingPoint) => F32,
(F64, I64) => F64,
(F64, FloatingPoint) => F64,
(U64, I64) => U64,
(U64, I32) => U64,
(StringType, _) => StringType, // meh, this all needs rigorous testing
(StringType, _) => StringType, // meh, this all needs rigorous testing. Update: this is in progress
_ => {
return Err(anyhow::anyhow!(
"Incompatible types. Expected {}, found {}",
@ -697,7 +700,16 @@ impl Expression {
Self::Grouping { expression, .. } => expression.infer_type(),
Self::Literal { literaltype, .. } => literaltype.clone(),
Self::List { literaltype, .. } => literaltype.clone(),
Self::Unary { right, .. } => right.infer_type(),
Self::Unary {
right, operator, ..
} => {
let literal_type = right.infer_type();
if literal_type == Integer && operator.token_type == Minus {
SignedInteger
} else {
UnsignedInteger
}
}
Self::Variable { var_type, .. } => var_type.clone(),
Self::FunctionCall { return_type, .. } => return_type.clone(),
}

6
src/bytecode_compiler.rs
View file

@ -9,7 +9,6 @@ use crate::vm::{
OP_GREATER_EQUAL, OP_LESS, OP_LESS_EQUAL, OP_MULTIPLY, OP_NEGATE, OP_NOT, OP_OR, OP_PRINT,
OP_RETURN, OP_SHL, OP_SHR, OP_SUBTRACT,
};
use anyhow::anyhow;
use std::collections::HashMap;
pub fn compile(
@ -47,7 +46,7 @@ pub(crate) fn compile_name(
struct Compiler {
chunk: Chunk,
had_error: bool,
_had_error: bool,
current_line: usize,
vars: HashMap<String, usize>,
}
@ -56,7 +55,7 @@ impl Compiler {
fn new(name: &str) -> Self {
Self {
chunk: Chunk::new(name),
had_error: false,
_had_error: false,
current_line: 0,
vars: HashMap::new(),
}
@ -142,7 +141,6 @@ impl Compiler {
} else {
name.clone()
};
println!("call {}", name);
let name_index = self
.chunk
.find_constant(&name)

94
src/compiler_tests.rs Normal file
View file

@ -0,0 +1,94 @@
#[cfg(test)]
mod tests {
use crate::compile;
use crate::scanner::scan;
#[test]
fn literal_int() {
assert!(compile("1").is_ok());
}
#[test]
fn literal_float() {
assert!(compile("2.1").is_ok());
}
#[test]
fn literal_float_scientific() {
assert!(compile("2.1e5").is_ok());
}
#[test]
fn literal_string() {
assert!(compile(r#""a""#).is_ok());
}
#[test]
fn literal_list() {
assert!(compile(r#"["abc","def"]"#).is_ok());
}
#[test]
fn let_infer_type() {
assert!(compile(r#"let a=1"#).is_ok());
}
#[test]
fn let_u32() {
assert!(compile(r#"let a:u32=1"#).is_ok());
}
#[test]
fn let_char() {
assert!(scan(r#"let a:char='a'"#).is_ok());
}
#[test]
fn let_u32_invalid_value_negative() {
let r = compile("let a:u32=-1");
assert!(r.is_err());
if let Err(e) = &r {
assert_eq!(
e.to_string(),
"error at line 1: Incompatible types. Expected u32, found i32/64"
);
}
}
#[test]
fn let_u64_invalid_value_negative() {
let r = compile("let a:u64=-1");
assert!(r.is_err());
if let Err(e) = &r {
assert_eq!(
e.to_string(),
"error at line 1: Incompatible types. Expected u64, found i32/64"
);
}
}
#[test]
fn let_u64_invalid_value_string() {
let r = compile(r#"let a:u64="not ok""#);
assert!(r.is_err());
if let Err(e) = &r {
assert_eq!(
e.to_string(),
"error at line 1: Incompatible types. Expected u64, found string"
);
}
}
#[test]
fn call_fn_with_args_returns_value() {
assert!(
compile(
r#"
fn hello(name: string) -> string:
"Hello " + name
hello("world")"#
)
.is_ok()
);
}
}

16
src/lib.rs
View file

@ -1,8 +1,20 @@
pub mod scanner;
use std::collections::HashMap;
use crate::scanner::scan;
pub mod ast_compiler;
pub mod bytecode_compiler;
pub mod vm;
pub mod chunk;
mod keywords;
pub mod scanner;
mod compiler_tests;
mod tokens;
mod value;
pub mod vm;
pub fn compile(src: &str) -> anyhow::Result<chunk::Chunk> {
let tokens = scan(src)?;
let mut registry = HashMap::new();
let ast= ast_compiler::compile(tokens)?;
let bytecode = bytecode_compiler::compile("", &ast, &mut registry)?;
Ok(bytecode)
}

37
src/main.rs
View file

@ -24,7 +24,7 @@ async fn main() -> anyhow::Result<()> {
if path.is_file() && path.ends_with("web.crud") {
print!("compiling {:?}: ", path);
let source = fs::read_to_string(path)?;
let tokens = scan(&source);
let tokens = scan(&source)?;
match ast_compiler::compile(tokens) {
Ok(statements) => {
let path = path
@ -53,7 +53,10 @@ async fn main() -> anyhow::Result<()> {
registry: registry.clone(),
});
println!("adding {}", path);
app = app.route(&format!("/{}",path.replace("/web", "")), get(handle_get).with_state(state.clone()));
app = app.route(
&format!("/{}", path.replace("/web", "")),
get(handle_get).with_state(state.clone()),
);
}
let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await?;
println!("listening on {}", listener.local_addr()?);
@ -77,33 +80,3 @@ async fn handle_get(State(state): State<Arc<AppState>>) -> Result<Json<String>,
))
}
//
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_compile() -> anyhow::Result<()> {
let tokens = scan(
r#"
fn hello(name: string) -> string:
"Hello "+name
hello("sander")"#,
);
let mut registry = HashMap::new();
match ast_compiler::compile(tokens) {
Ok(statements) => {
println!("{:?}", statements);
let chunk = compile("", &statements, &mut registry)?;
chunk.disassemble();
// println!("{}", interpret(&chunk).await?);
}
Err(e) => {
println!("{}", e)
}
}
println!("{:?}", registry);
Ok(())
}
}

49
src/scanner.rs
View file

@ -1,3 +1,4 @@
use anyhow::anyhow;
use crate::tokens::TokenType::{BitXor, FloatingPoint, Integer};
use crate::{
keywords,
@ -7,7 +8,7 @@ use crate::{
},
};
pub fn scan(source: &str) -> Vec<Token> {
pub fn scan(source: &str) -> anyhow::Result<Vec<Token>> {
let scanner = Scanner {
chars: source.chars().collect(),
current: 0,
@ -20,17 +21,17 @@ pub fn scan(source: &str) -> Vec<Token> {
}
impl Scanner {
fn scan(mut self) -> Vec<Token> {
fn scan(mut self) -> anyhow::Result<Vec<Token>> {
while !self.is_at_end() {
self.start = self.current;
self.scan_token();
self.scan_token()?;
}
self.add_token(TokenType::Eol);
self.add_token(TokenType::Eof);
self.tokens
Ok(self.tokens)
}
fn scan_token(&mut self) {
fn scan_token(&mut self) -> anyhow::Result<()>{
let c = self.advance();
if self.new_line && (c == ' ' || c == '\t') {
self.add_token(TokenType::Indent);
@ -106,7 +107,8 @@ impl Scanner {
self.add_token(TokenType::Slash);
}
}
'"' => self.string(),
'\'' => self.char()?,
'"' => self.string()?,
'\r' | '\t' | ' ' => {}
'\n' => {
self.line += 1;
@ -136,11 +138,12 @@ impl Scanner {
} else if is_alpha(c) {
self.identifier();
} else {
println!("Unexpected identifier at line {}", self.line);
return Err(anyhow!("Unexpected identifier at line {}", self.line));
}
}
}
}
Ok(())
}
fn identifier(&mut self) {
@ -163,14 +166,35 @@ impl Scanner {
self.advance();
}
while is_digit(self.peek()) {
while is_digit_or_scientific(self.peek()) {
self.advance();
}
let value: String = self.chars[self.start..self.current].iter().collect();
self.add_token_with_value(if has_dot { FloatingPoint } else { Integer }, value);
}
fn string(&mut self) {
fn char(&mut self) -> anyhow::Result<()>{
while self.peek() != '\'' && !self.is_at_end() {
self.advance();
}
if self.is_at_end() {
return Err(anyhow!("Unterminated char at {}", self.line))
}
self.advance();
let value: String = self.chars[self.start + 1..self.current - 1]
.iter()
.collect();
if value.len() != 1 {
return Err(anyhow!("Illegal char length for {} at line {}", value, self.line))
}
self.add_token_with_value(TokenType::Char, value);
Ok(())
}
fn string(&mut self) -> anyhow::Result<()>{
while self.peek() != '"' && !self.is_at_end() {
if self.peek() == '\n' {
self.line += 1;
@ -179,7 +203,7 @@ impl Scanner {
}
if self.is_at_end() {
println!("Unterminated string at {}", self.line)
return Err(anyhow!("Unterminated string at {}", self.line))
}
self.advance();
@ -188,6 +212,7 @@ impl Scanner {
.iter()
.collect();
self.add_token_with_value(TokenType::StringType, value);
Ok(())
}
fn peek(&self) -> char {
@ -244,6 +269,10 @@ fn is_digit(c: char) -> bool {
c >= '0' && c <= '9'
}
fn is_digit_or_scientific(c: char) -> bool {
is_digit(c) || c=='e' || c=='E'
}
fn is_alphanumeric(c: char) -> bool {
is_alpha(c) || is_digit(c)
}

13
src/tokens.rs
View file

@ -17,11 +17,6 @@ impl Token {
}
}
#[derive(Debug)]
enum Value {
None,
}
#[derive(Debug, PartialEq, Clone, Copy, Hash)]
pub enum TokenType {
Bang,
@ -55,7 +50,9 @@ pub enum TokenType {
Identifier,
If,
Indent,
Integer, //undetermined integer type
Integer,
SignedInteger,
UnsignedInteger,
LeftBrace,
LeftBracket,
LeftParen,
@ -69,7 +66,7 @@ pub enum TokenType {
LogicalOr,
Minus,
Not,
FloatingPoint, //undetermined float type
FloatingPoint,
Object,
Plus,
Print,
@ -157,6 +154,8 @@ impl fmt::Display for TokenType {
TokenType::True => write!(f, "true"),
TokenType::Void => write!(f, "()"),
TokenType::While => write!(f, "while"),
TokenType::SignedInteger => write!(f, "i32/64"),
TokenType::UnsignedInteger => write!(f, "u32/64"),
}
}
}

4
src/value.rs
View file

@ -143,7 +143,7 @@ impl Display for Value {
&Value::Enum => write!(f, "enum"),
&Value::Struct(v) => write!(f, "{}", v),
&Value::List(v) => write!(f, "{:?}", v),
&Value::Map(v) => write!(f, "map"),
&Value::Map(_) => write!(f, "map"),
&Value::Error(v) => write!(f, "{}", v),
&Value::Void => write!(f, "()"),
}
@ -295,7 +295,7 @@ impl Not for &Value {
type Output = anyhow::Result<Value>;
fn not(self) -> Self::Output {
match (self) {
match self {
Value::Bool(b) => Ok(Value::Bool(!b)),
Value::I32(i32) => Ok(Value::I32(!i32)),
Value::I64(i64) => Ok(Value::I64(!i64)),

5
src/vm.rs
View file

@ -1,7 +1,6 @@
use crate::chunk::Chunk;
use crate::value::Value;
use anyhow::anyhow;
use bumpalo::Bump;
use std::collections::HashMap;
use tracing::debug;
@ -175,10 +174,6 @@ impl <'a> Vm<'a> {
chunk.constants[index].to_string() //string??
}
fn reset_stack(&mut self) {
self.stack.clear();
}
fn push(&mut self, value: Value) {
self.stack.push(value);
}