some clarifications

Cargo.toml

@@ -1,9 +1,14 @@
 [package]
 name = "sqlighters"
+description = "Easy creation of SQLite binary format for use in serverside applications, where typically the client runs SQLite in WASM"
 version = "0.1.0"
 edition = "2021"
-
+homepage = "https://gitlab.com/sander-hautvast/sqlighte.rs"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+repository = "https://gitlab.com/sander-hautvast/sqlighte.rs.git"
+license-file = "LICENSE"
+readme = "README.md"
+keywords = ["sqlite"]
+categories = ["database", "WASM"]
 
 [dependencies]
 byteorder = "1.4.3"

README.md

@@ -1,9 +1,35 @@
 # Sqlighte.Rs
 
-* rust version of https://gitlab.com/sander-hautvast/sqlighter
+https://crates.io/crates/sqlighters
-* still work in progress
 
-Creating a database will be as simple as: 
+**Sqlighter**
+* Inspired by a new feature in .Net blazor (see https://www.youtube.com/watch?v=lP_qdhAHFlg&t=300s)
+* Creates a SQLite database file from any tabular data.
+* So instead of a rest api serving json, enables binary download of data in native SQLite format
+* So that SQLite in running in WASM (so in the browser) can be used to query the data.
+
+**Why not use the official SQLite library serverside for this purpose?**
+
+*excellent question!*
+
+But, then I would have to first create and SQLite database and fill it with results and then load the database file and serve it for http requests. While this should also work, it sounds as more overhead. In Sqlighter the data stays in memory. (yes, that's a problem if the data gets reallly BIG; considering offload to file)
+
+**Usable when:**
+* you have quite a lot of (tabular) data, that is read-only, or does not need to be (ultra) realtime.
+* and your users need to quickly apply different search criteria on it.
+* Using Sqlighter avoids server roundtrips and improves the user experience.
+* Bear in mind that, while you, as a developer, cannot directly read the payload, like JSON allows, SQLite is available on pretty much any platform,
+  and then you can leverage the power of SQL to inspect the data.
+
+* Thing to note: Sqlite is really relaxed when it comes to schema validation.
+  That means that 2 records in the same table can contain values of totally different types(!). The number of values can also vary. All perfectly legal from the standpoint of Sqlighter.
+  And maybe not when writing to Sqlite itself, but perfectly readable!
+
+**About the name**
+* It lights up an SQLite database :)
+
+
+Creating a database is as simple as: 
 ```rust
 fn test_build() -> Result<(), Error> {
         let mut builder = Builder::new();
@@ -18,7 +44,17 @@ fn test_build() -> Result<(), Error> {
         let database: Database = builder.into();
         let file = File::create("foo.db")?;
         let writer = BufWriter::new(file);
-        write(database, writer)?;
+        write_sqlite(database, writer)?;
         Ok(())
     }
-```
+```
+
+**Be aware**
+* The schema and the actual data don't have to match! But that is how SQLite itself also works, pretty much.
+* And: 2 records in the same table can contain values of totally different types(!). The number of values can also vary. All perfectly legal from the standpoint of Sqlighter.
+  And maybe not when writing to SQLite itself (using sql), but perfectly readable from the file.
+   
+
+
+**Current status**
+* It works for tables of any size. Indexes are not supported, but you can always add them client-side.

src/builder.rs

@@ -1,13 +1,13 @@
-use std::mem;
+use crate::database::SchemaRecord;
-use crate::database::{Database, SchemaRecord};
 use crate::page::{self, Page};
 use crate::record::Record;
+use std::mem;
 
-pub struct Builder {
+pub struct DatabaseBuilder {
-    current_page: Page,
+    pub current_page: Page,
-    n_records_on_current_page: u16,
+    pub n_records_on_current_page: u16,
-    leaf_pages: Vec<Page>,
+    pub leaf_pages: Vec<Page>,
-    schema: Option<SchemaRecord>,
+    pub schema: Option<SchemaRecord>,
 }
 
 fn new_page() -> Page {
@@ -16,7 +16,7 @@ fn new_page() -> Page {
     page
 }
 
-impl Builder {
+impl DatabaseBuilder {
     pub fn new() -> Self {
         Self {
             current_page: new_page(),
@@ -29,14 +29,15 @@ impl Builder {
     pub fn add_record(&mut self, record: Record) {
         if self.current_page_is_full(&record) {
             self.finish_current_page();
-            self.leaf_pages.push(mem::replace(&mut self.current_page, new_page()));
+            self.leaf_pages
+                .push(mem::replace(&mut self.current_page, new_page()));
             self.n_records_on_current_page = 0;
         }
 
         self.current_page.key = record.rowid; //clone?
         let bytes: Vec<u8> = record.into();
         self.current_page.put_bytes_bw(&bytes);
-        self.current_page.put_u16(self.current_page.bw_position as u16);
+        self.current_page.put_u16(self.current_page.bw_position);
         self.n_records_on_current_page += 1;
     }
 
@@ -54,46 +55,3 @@ impl Builder {
         self.current_page.put_u16(self.current_page.bw_position);
     }
 }
-
-impl Into<Database> for Builder {
-    fn into(mut self) -> Database {
-        self.current_page.fw_position = page::POSITION_CELL_COUNT;
-        self.current_page.put_u16(self.n_records_on_current_page);
-
-        if self.n_records_on_current_page > 0 {
-            self.current_page.put_u16(self.current_page.bw_position);
-        } else {
-            self.current_page.put_u16(self.current_page.bw_position - 1);
-        }
-
-        self.leaf_pages.push(self.current_page);
-        Database::new(self.schema.unwrap(), self.leaf_pages) //panics is schema is not set
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::fs::File;
-    use std::io::{BufWriter, Error};
-    use crate::database::write;
-    use crate::values;
-    use super::*;
-
-    #[test]
-    fn test_build() -> Result<(), Error> {
-        let mut builder = Builder::new();
-        builder.schema(
-            "foo",
-            "create table foo(bar varchar(10))",
-        );
-        let mut record = Record::new(1);
-        record.add_value(values::string("helloworld"));
-        builder.add_record(record);
-
-        let database: Database = builder.into();
-        let file = File::create("foo.db")?;
-        let writer = BufWriter::new(file);
-        write(database, writer)?;
-        Ok(())
-    }
-}

src/database.rs

@@ -1,10 +1,10 @@
-use std::io::{BufWriter, Error, Write};
+use crate::builder::DatabaseBuilder;
-use std::mem;
-use crate::varint;
 use crate::page;
 use crate::page::{Page, PageType};
 use crate::record::Record;
-use crate::values;
+use crate::varint;
+use std::io::{BufWriter, Error, Write};
+use std::mem;
 
 pub struct Database {
     schema: SchemaRecord,
@@ -13,18 +13,177 @@ pub struct Database {
 
 impl Database {
     pub fn new(schema: SchemaRecord, leaf_pages: Vec<Page>) -> Self {
-        Self {
+        Self { schema, leaf_pages }
-            schema,
-            leaf_pages,
-        }
     }
 }
 
+impl From<DatabaseBuilder> for Database {
+    fn from(mut dbb: DatabaseBuilder) -> Self {
+        dbb.current_page.fw_position = page::POSITION_CELL_COUNT;
+        dbb.current_page.put_u16(dbb.n_records_on_current_page);
+
+        if dbb.n_records_on_current_page > 0 {
+            dbb.current_page.put_u16(dbb.current_page.bw_position);
+        } else {
+            dbb.current_page.put_u16(dbb.current_page.bw_position - 1);
+        }
+
+        dbb.leaf_pages.push(dbb.current_page);
+        Database::new(dbb.schema.unwrap_or_default(), dbb.leaf_pages)
+    }
+}
+
+pub fn write_sqlite<W: Write>(database: Database, mut writer: BufWriter<W>) -> Result<(), Error> {
+    let mut current_top_layer = database.leaf_pages;
+    let mut n_pages = current_top_layer.len();
+    while current_top_layer.len() > 1 {
+        // db needs interior pages?
+        current_top_layer = create_interior_pages(current_top_layer);
+        n_pages += current_top_layer.len();
+    }
+
+    // assert(won't panic)
+    let table_root_page = current_top_layer.get_mut(0).unwrap();
+    writer.write_all(&create_header_page((n_pages + 1) as u32, database.schema).data)?; // 1 for header page
+
+    set_childrefs_write(table_root_page, &mut writer, 3)
+}
+
+fn set_childrefs_write<W: Write>(
+    page: &mut Page,
+    writer: &mut BufWriter<W>,
+    mut page_counter: u32,
+) -> Result<(), Error> {
+    if let PageType::Interior = page.page_type {
+        page.fw_position = page::POSITION_CELL_COUNT;
+        page.put_u16((page.children.len() - 1) as u16);
+
+        for index in 0..page.children.len() - 1 {
+            page.fw_position = page::START_OF_INTERIOR_PAGE + (index as u16) * 2;
+            page.fw_position = page.get_u16();
+            page.put_u32(page_counter);
+            page_counter += 1;
+        }
+
+        page.fw_position = page::POSITION_RIGHTMOST_POINTER_LEAFPAGES;
+        page.put_u32(page_counter);
+        page_counter += 1;
+    }
+
+    writer.write_all(&page.data)?;
+
+    for child in &mut page.children {
+        set_childrefs_write(child, writer, page_counter)?;
+    }
+    Ok(())
+}
+
+fn create_header_page(n_pages: u32, schema: SchemaRecord) -> Page {
+    let mut header_page = Page::new_root();
+    write_header(&mut header_page, n_pages);
+
+    let payload_location_write_location = header_page.fw_position; // mark current position
+
+    let payload_location = write_schema(&mut header_page, schema); //write schema payload from the end
+    header_page.fw_position = payload_location_write_location; // go back to marked position
+    header_page.put_u16(payload_location); //payload start
+    header_page.put_u8(0); // the number of fragmented free bytes within the cell content area
+    header_page.put_u16(payload_location); // first cell
+    header_page
+}
+
+fn write_schema(root_page: &mut Page, schema_record: SchemaRecord) -> u16 {
+    let record: Record = schema_record.into();
+    let bytes: Vec<u8> = record.into();
+    root_page.put_bytes_bw(&bytes);
+    root_page.bw_position
+}
+
+fn create_interior_pages(child_pages: Vec<Page>) -> Vec<Page> {
+    let mut interior_pages = Vec::new();
+    let mut interior_page = Page::new_interior();
+    interior_page.key = child_pages.iter().map(|p| p.key).max().unwrap_or(0);
+    interior_page.fw_position = page::START_OF_INTERIOR_PAGE;
+    let children_length = child_pages.len();
+    let mut last_leaf: Page = Page::new_leaf(); // have to assign :(
+    for (child_count, leaf_page) in child_pages.into_iter().enumerate() {
+        if child_count < children_length - 1 {
+            if interior_page.bw_position <= interior_page.fw_position + 15 {
+                // 15 is somewhat arbitrary, but safe
+                interior_page.fw_position = page::START_OF_CONTENT_AREA;
+                interior_page.put_u16(interior_page.bw_position);
+                interior_page.fw_position += 5;
+                interior_pages.push(mem::replace(&mut interior_page, Page::new_interior()));
+                interior_page.fw_position = page::START_OF_INTERIOR_PAGE;
+            }
+            create_cell(&mut interior_page, &leaf_page);
+            interior_page.add_child(leaf_page);
+        } else {
+            last_leaf = leaf_page;
+        }
+    }
+
+    interior_page.fw_position = page::START_OF_CONTENT_AREA;
+    interior_page.put_u16(interior_page.bw_position);
+    interior_page.fw_position += 5;
+    interior_page.add_child(last_leaf);
+    interior_pages.push(interior_page);
+    interior_pages
+}
+
+fn create_cell(interior_page: &mut Page, child_page: &Page) {
+    let mut cell: Vec<u8> = vec![0; 5];
+    cell.append(&mut varint::write(child_page.key));
+
+    interior_page.put_bytes_bw(&cell);
+    interior_page.put_u16(interior_page.bw_position);
+}
+
+fn write_header(rootpage: &mut Page, n_pages: u32) {
+    rootpage.put_bytes(&MAGIC_HEADER);
+    rootpage.put_u16(DEFAULT_PAGE_SIZE);
+    rootpage.put_u8(FILE_FORMAT_WRITE_VERSION);
+    rootpage.put_u8(FILE_FORMAT_READ_VERSION);
+    rootpage.put_u8(RESERVED_SIZE);
+    rootpage.put_u8(MAX_EMBED_PAYLOAD_FRACTION);
+    rootpage.put_u8(MIN_EMBED_PAYLOAD_FRACTION);
+    rootpage.put_u8(LEAF_PAYLOAD_FRACTION);
+    rootpage.put_u32(FILECHANGE_COUNTER);
+    rootpage.put_u32(n_pages); // file size in pages
+    rootpage.put_u32(FREELIST_TRUNK_PAGE_HUMBER); // Page number of the first freelist trunk page.
+    rootpage.put_u32(TOTAL_N_FREELIST_PAGES);
+    rootpage.put_u32(SCHEMA_COOKIE);
+    rootpage.put_u32(SQLITE_SCHEMAVERSION);
+    rootpage.put_u32(SUGGESTED_CACHESIZE);
+    rootpage.put_u32(LARGEST_ROOT_BTREE_PAGE);
+    rootpage.put_u32(ENCODING_UTF8);
+    rootpage.put_u32(USER_VERSION);
+    rootpage.put_u32(VACUUM_MODE_OFF); // True (non-zero) for incremental-vacuum mode. False (zero) otherwise.
+    rootpage.put_u32(APP_ID); // Application ID
+    rootpage.put_bytes(&FILLER); // Reserved for expansion. Must be zero.
+    rootpage.put_bytes(&VERSION_VALID_FOR); // The version-valid-for number
+    rootpage.put_bytes(&SQLITE_VERSION); // SQLITE_VERSION_NUMBER
+    rootpage.put_u8(TABLE_LEAF_PAGE); // leaf table b-tree page for schema
+    rootpage.put_u16(NO_FREE_BLOCKS); // zero if there are no freeblocks
+    rootpage.put_u16(1); // the number of cells on this page
+}
+
 pub struct SchemaRecord {
-    rowid: u64,
+    pub rowid: u64,
-    table_name: String,
+    pub table_name: String,
-    root_page: u32,
+    pub root_page: u32,
-    sql: String,
+    pub sql: String,
+}
+
+impl Default for SchemaRecord {
+    fn default() -> Self {
+        Self {
+            rowid: 0,
+            table_name: "".to_owned(),
+            root_page: 3,
+            sql: "".to_owned(),
+        }
+    }
 }
 
 impl SchemaRecord {
@@ -38,172 +197,9 @@ impl SchemaRecord {
     }
 }
 
-impl Into<Record> for SchemaRecord {
+const MAGIC_HEADER: [u8; 16] = [
-    fn into(self) -> Record {
+    0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00,
-        let mut record = Record::new(self.rowid);
+];
-        record.add_value(values::string("table"));
-        record.add_value(values::string(&self.table_name.to_ascii_lowercase()));
-        record.add_value(values::string(&self.table_name.to_ascii_lowercase()));
-        record.add_value(values::integer(self.root_page as i64));
-        record.add_value(values::string(&self.sql));
-        record
-    }
-}
-
-pub fn write<W: Write>(database: Database, mut writer: BufWriter<W>) -> Result<(), Error> {
-    let mut current_top_layer = database.leaf_pages;
-    let mut n_pages = current_top_layer.len();
-    while current_top_layer.len() > 1 { // db needs interior pages?
-        current_top_layer = create_interior_pages(current_top_layer);
-        n_pages += current_top_layer.len();
-    }
-
-    let table_root_page = current_top_layer.get_mut(0).unwrap();
-    writer.write_all(&create_header_page(n_pages + 1, database.schema).data)?; // 1 for header page
-
-    assign_pagenumbers(table_root_page, 2);
-    set_page_references(table_root_page);
-    write_pages(&mut writer, table_root_page)?;
-
-    Ok(())
-}
-
-fn assign_pagenumbers(page: &mut Page, page_counter: u32) {
-    page.number = page_counter;
-    let mut counter = page_counter;
-    for child in page.children.iter_mut() {
-        counter += 1;
-        assign_pagenumbers(child, counter);
-    }
-}
-
-fn set_page_references(page: &mut Page) {
-    if let PageType::Interior = page.page_type {
-        page.fw_position = page::POSITION_CELL_COUNT;
-        page.put_u16((page.children.len() - 1) as u16);
-
-        page.fw_position = page::POSITION_RIGHTMOST_POINTER_LEAFPAGES;
-        page.put_u32(page.get_page_nr_last_child());
-
-        let index = 0;
-        let before_last = page.children.len() - 1;
-        let child_numbers: Vec<u32> = page.children.iter().map(|child| child.number).collect();
-
-        for child_page_number in child_numbers {
-            if index < before_last {
-                page.fw_position = page::START_OF_INTERIOR_PAGE + (index as u16) * 2;
-                page.fw_position = page.get_u16();
-                page.put_u32(child_page_number);
-            }
-        }
-    }
-    for child in page.children.iter_mut() {
-        set_page_references(child);
-    }
-}
-
-fn write_pages<W: Write>(writer: &mut BufWriter<W>, page: &Page) -> Result<(), Error> {
-    writer.write(&page.data)?;
-    for child in page.children.iter() {
-        write_pages(writer, child)?;
-    }
-    Ok(())
-}
-
-fn create_header_page(n_pages: usize, schema: SchemaRecord) -> Page {
-    let mut header_page = Page::new_root();
-    write_header(&mut header_page, n_pages as u32);
-
-    let payload_location_write_location = header_page.fw_position; // mark current position
-
-    let payload_location = write_schema(&mut header_page, schema); //write schema payload from the end
-    header_page.fw_position = payload_location_write_location; // go back to marked position
-    header_page.put_u16(payload_location); //payload start
-    header_page.put_u8(0); // the number of fragmented free bytes within the cell content area
-    header_page.put_u16(payload_location); // first cell
-    return header_page;
-}
-
-fn write_schema(root_page: &mut Page, schema_record: SchemaRecord) -> u16 {
-    let record: Record = schema_record.into();
-    let bytes: Vec<u8> = record.into();
-    root_page.put_bytes_bw(&bytes);
-    root_page.bw_position
-}
-
-fn create_interior_pages(mut child_pages: Vec<Page>) -> Vec<Page> {
-    let mut interior_pages = Vec::new();
-    let mut interior_page = Page::new_interior();
-    interior_page.key = child_pages.iter().map(|p| p.key).max().unwrap();
-    interior_page.fw_position = page::START_OF_INTERIOR_PAGE;
-    let mut page_index = 0;
-    let children_length = child_pages.len();
-    let mut child_count = 0;
-    let mut last_leaf: Page = Page::new_leaf(); // have to assign :(
-    for mut leaf_page in child_pages {
-        if child_count < children_length - 1 {
-            if interior_page.bw_position <= interior_page.fw_position + 15 { // 15 is somewhat arbitrary
-                interior_page.fw_position = page::START_OF_CONTENT_AREA;
-                interior_page.put_u16(interior_page.bw_position);
-                interior_page.put_bytes(&[0, 0, 0, 0, 0]);
-
-                interior_pages.push(mem::replace(&mut interior_page, Page::new_interior()));
-                interior_page.fw_position = page::START_OF_INTERIOR_PAGE;
-            }
-            create_cell(&mut leaf_page);
-            interior_page.add_child(leaf_page);
-            page_index += 1;
-        } else {
-            last_leaf = leaf_page;
-        }
-        child_count += 1;
-    }
-
-    interior_page.fw_position = page::START_OF_CONTENT_AREA;
-    interior_page.put_u16(interior_page.bw_position);
-    interior_page.put_bytes(&[0, 0, 0, 0, 0]);
-    interior_page.add_child(last_leaf);
-    interior_pages.push(interior_page);
-    interior_pages
-}
-
-fn create_cell(page: &mut Page) {
-    let mut cell: Vec<u8> = vec![0, 0, 0, 0]; // not an expensive call right?
-    cell.append(&mut varint::write(page.key));
-    page.put_bytes_bw(&cell);
-    page.put_u16(page.bw_position);
-}
-
-fn write_header(rootpage: &mut Page, n_pages: u32) {
-    rootpage.put_bytes(&MAGIC_HEADER);
-    rootpage.put_u16(DEFAULT_PAGE_SIZE);
-    rootpage.put_u8(FILE_FORMAT_WRITE_VERSION);
-    rootpage.put_u8(FILE_FORMAT_READ_VERSION);
-    rootpage.put_u8(RESERVED_SIZE);
-    rootpage.put_u8(MAX_EMBED_PAYLOAD_FRACTION);
-    rootpage.put_u8(MIN_EMBED_PAYLOAD_FRACTION);
-    rootpage.put_u8(LEAF_PAYLOAD_FRACTION);
-    rootpage.put_u32(FILECHANGE_COUNTER);
-    rootpage.put_u32(n_pages);// file size in pages
-    rootpage.put_u32(FREELIST_TRUNK_PAGE_HUMBER);// Page number of the first freelist trunk page.
-    rootpage.put_u32(TOTAL_N_FREELIST_PAGES);
-    rootpage.put_u32(SCHEMA_COOKIE);
-    rootpage.put_u32(SQLITE_SCHEMAVERSION);
-    rootpage.put_u32(SUGGESTED_CACHESIZE);
-    rootpage.put_u32(LARGEST_ROOT_BTREE_PAGE);
-    rootpage.put_u32(ENCODING_UTF8);
-    rootpage.put_u32(USER_VERSION);
-    rootpage.put_u32(VACUUM_MODE_OFF);// True (non-zero) for incremental-vacuum mode. False (zero) otherwise.
-    rootpage.put_u32(APP_ID);// Application ID
-    rootpage.put_bytes(&FILLER);// Reserved for expansion. Must be zero.
-    rootpage.put_bytes(&VERSION_VALID_FOR);// The version-valid-for number
-    rootpage.put_bytes(&SQLITE_VERSION);// SQLITE_VERSION_NUMBER
-    rootpage.put_u8(TABLE_LEAF_PAGE); // leaf table b-tree page for schema
-    rootpage.put_u16(NO_FREE_BLOCKS); // zero if there are no freeblocks
-    rootpage.put_u16(1); // the number of cells on this page
-}
-
-const MAGIC_HEADER: [u8; 16] = [0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00];
 pub const DEFAULT_PAGE_SIZE: u16 = 4096;
 const FILE_FORMAT_WRITE_VERSION: u8 = 1;
 const FILE_FORMAT_READ_VERSION: u8 = 1;
@@ -222,7 +218,10 @@ const ENCODING_UTF8: u32 = 1;
 const USER_VERSION: u32 = 0;
 const VACUUM_MODE_OFF: u32 = 0;
 const APP_ID: u32 = 0;
-const FILLER: [u8; 20] = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
+const FILLER: [u8; 20] = [
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00,
+];
 const VERSION_VALID_FOR: [u8; 4] = [0, 0, 0x03, 250];
 const SQLITE_VERSION: [u8; 4] = [0x00, 0x2e, 0x5F, 0x1A];
 const NO_FREE_BLOCKS: u16 = 0;
@@ -230,5 +229,3 @@ pub const TABLE_LEAF_PAGE: u8 = 0x0d;
 pub const TABLE_INTERIOR_PAGE: u8 = 0x05;
 const INDEX_LEAF_PAGE: u8 = 0x0a;
 const INDEX_INTERIOR_PAGE: u8 = 0x02;
-
-

src/lib.rs

@@ -1,18 +1,34 @@
 #![allow(dead_code)]
 
-mod page;
+mod builder;
 mod database;
+mod page;
+mod record;
 mod values;
 mod varint;
-mod record;
-mod builder;
-
 
 #[cfg(test)]
 mod tests {
+    use crate::builder::DatabaseBuilder;
+    use crate::database::{write_sqlite, Database};
+    use crate::record::Record;
+    use crate::values;
+    use std::fs::File;
+    use std::io::{BufWriter, Error};
+
     #[test]
-    fn it_works() {
+    fn test_build() -> Result<(), Error> {
-        let result = 2 + 2;
+        let mut builder = DatabaseBuilder::new();
-        assert_eq!(result, 4);
+        builder.schema("foo", "create table foo(bar varchar(10))");
+        for i in 0..10000 {
+            let mut record = Record::new(i);
+            record.add_value(values::string("helloworld"));
+            builder.add_record(record);
+        }
+        let database: Database = builder.into();
+        let file = File::create("foo.db")?;
+        let writer = BufWriter::new(file);
+        write_sqlite(database, writer)?;
+        Ok(())
     }
 }

src/page.rs

@@ -1,5 +1,5 @@
-use byteorder::{BigEndian, ByteOrder};
 use crate::database;
+use byteorder::{BigEndian, ByteOrder};
 
 pub const POSITION_CELL_COUNT: u16 = 3;
 pub const START_OF_CONTENT_AREA: u16 = 5;
@@ -13,14 +13,13 @@ pub enum PageType {
     Other,
 }
 
-/// Represents an SQLite page
+/// Represents an `SQLite` page
 pub struct Page {
     pub data: Vec<u8>,
     pub fw_position: u16,
     pub bw_position: u16,
     pub key: u64,
     pub children: Vec<Page>,
-    pub number: u32,
     pub page_type: PageType,
 }
 
@@ -32,7 +31,6 @@ impl Page {
             bw_position: size,
             key: 0,
             children: Vec::new(),
-            number: 0,
             page_type,
         }
     }
@@ -44,13 +42,12 @@ impl Page {
             bw_position: size as u16,
             key: 0,
             children: Vec::new(),
-            number: 0,
             page_type: PageType::Other,
         }
     }
 
     pub fn new_root() -> Self {
-        Page::with_capacity(database::DEFAULT_PAGE_SIZE, PageType::Leaf)
+        Page::with_capacity(database::DEFAULT_PAGE_SIZE, PageType::Other)
     }
 
     pub fn new_leaf() -> Self {
@@ -61,7 +58,7 @@ impl Page {
 
     pub fn new_interior() -> Self {
         let mut page = Page::with_capacity(database::DEFAULT_PAGE_SIZE, PageType::Interior);
-        page.put_u8(database::TABLE_LEAF_PAGE);
+        page.put_u8(database::TABLE_INTERIOR_PAGE);
         page
     }
 
@@ -109,14 +106,9 @@ impl Page {
         self.put_bytes_bw(&u32_to_bytes(value));
     }
 
-    // may panic
-    pub fn get_page_nr_last_child(&self) -> u32 {
-        self.children[self.children.len() - 1].number
-    }
-
     pub fn get_u16(&self) -> u16 {
         let position = self.fw_position as usize;
-        (self.data[position] as u16) << 8 + self.data[position + 1]
+        (u16::from(self.data[position]) << 8) + u16::from(self.data[position + 1])
         // does not increase the fw pointerr
     }
 }
@@ -218,4 +210,4 @@ mod tests {
         assert_eq!(b.data[7], 0x10);
         assert_eq!(b.data[8], 0x02);
     }
-}
+}

src/record.rs

@@ -1,4 +1,5 @@
-use crate::values::*;
+use crate::database::SchemaRecord;
+use crate::values::{integer, string, Value};
 use crate::varint;
 
 pub struct Record {
@@ -8,6 +9,7 @@ pub struct Record {
 }
 
 impl Record {
+    /// Users have to supply the rowid. In the java version this can be done automatically
     pub fn new(rowid: u64) -> Self {
         Self {
             rowid,
@@ -21,42 +23,54 @@ impl Record {
 
     /// length of the byte representation
     pub fn bytes_len(&self) -> u16 {
-        let record_length: u16 = self.values.iter()
+        let record_length: u16 = self.values.iter().map(Value::len).sum();
-            .map(|v| v.len())
+        record_length + 1
-            .sum();
-        record_length
     }
 }
 
-impl Into<Vec<u8>> for Record{
+impl From<Record> for Vec<u8> {
-    fn into(mut self) -> Vec<u8> {
+    fn from(mut record: Record) -> Vec<u8> {
-        let record_length = self.bytes_len();
+        let record_length = record.bytes_len();
-        let mut length_bytes = varint::write(record_length as u64);
+        let mut length_bytes = varint::write(u64::from(record_length));
-        let mut rowid_bytes = varint::write(self.rowid);
+        let mut rowid_bytes = varint::write(record.rowid);
 
-        let mut buffer = Vec::with_capacity(length_bytes.len() + rowid_bytes.len() + record_length as usize);
+        let mut buffer =
+            Vec::with_capacity(length_bytes.len() + rowid_bytes.len() + record_length as usize);
         buffer.append(&mut length_bytes);
         buffer.append(&mut rowid_bytes);
 
         // 'The initial portion of the payload that does not spill to overflow pages.'
-        let length_of_encoded_column_types: usize = self.values.iter()
+        let length_of_encoded_column_types: usize =
-            .map(|v| v.datatype.len())
+            record.values.iter().map(|v| v.datatype.len()).sum();
-            .sum();
+        buffer.append(&mut varint::write(
-        buffer.append(&mut varint::write((length_of_encoded_column_types + 1) as u64));
+            (length_of_encoded_column_types + 1) as u64,
+        ));
 
         //write all types
-        for v in self.values.iter_mut() {
+        for v in &mut record.values {
-            buffer.append(&mut v.datatype)
+            buffer.append(&mut v.datatype);
         }
 
         //  write all values
-        for v in self.values.iter_mut() {
+        for v in &mut record.values {
-            buffer.append(&mut v.data)
+            buffer.append(&mut v.data);
         }
         buffer
     }
 }
 
+impl From<SchemaRecord> for Record {
+    fn from(s: SchemaRecord) -> Self {
+        let mut record = Record::new(s.rowid);
+        record.add_value(string("table"));
+        record.add_value(string(&s.table_name.to_ascii_lowercase()));
+        record.add_value(string(&s.table_name.to_ascii_lowercase()));
+        record.add_value(integer(i64::from(s.root_page)));
+        record.add_value(string(&s.sql));
+        record
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -65,5 +79,7 @@ mod tests {
     fn test() {
         let mut record = Record::new(1);
         record.add_value(string("hello"));
+        let bytes: Vec<u8> = record.into();
+        assert_eq!(bytes, vec![7, 1, 2, 23, 104, 101, 108, 108, 111]);
     }
-}
+}

src/values.rs

@@ -1,12 +1,11 @@
-use byteorder::{BigEndian, ByteOrder};
 use crate::varint;
+use byteorder::{BigEndian, ByteOrder};
 
 pub struct Value {
     pub datatype: Vec<u8>,
     pub data: Vec<u8>,
 }
 
-
 impl Value {
     pub fn len(&self) -> u16 {
         (self.datatype.len() + self.data.len()) as u16
@@ -15,21 +14,33 @@ impl Value {
 
 pub fn string(value: &str) -> Value {
     let bytes = value.chars().map(|c| c as u8).collect::<Vec<_>>();
-    Value { datatype: varint::write((bytes.len() * 2 + 13) as u64), data: bytes }
+    Value {
+        datatype: varint::write((bytes.len() * 2 + 13) as u64),
+        data: bytes,
+    }
 }
 
 pub fn blob(value: Vec<u8>) -> Value {
-    Value { datatype: varint::write((value.len() * 2 + 12) as u64), data: value }
+    Value {
+        datatype: varint::write((value.len() * 2 + 12) as u64),
+        data: value,
+    }
 }
 
 pub fn integer(value: i64) -> Value {
-    Value { datatype: get_int_type(value), data: sqlite_integer_to_bytes(value) }
+    Value {
+        datatype: get_int_type(value),
+        data: sqlite_integer_to_bytes(value),
+    }
 }
 
 pub fn float(value: f64) -> Value {
     let mut buffer = [0_u8; 8];
     BigEndian::write_f64(&mut buffer, value);
-    Value { datatype: vec![7], data: buffer.to_vec() }
+    Value {
+        datatype: vec![7],
+        data: buffer.to_vec(),
+    }
 }
 
 pub fn len(value: &Value) -> usize {
@@ -46,10 +57,10 @@ fn sqlite_integer_to_bytes(value: i64) -> Vec<u8> {
     }
 }
 
-fn i64_to_bytes(n: i64, nbytes: u8) -> Vec<u8> {
+fn i64_to_bytes(value: i64, len: u8) -> Vec<u8> {
     let mut bytes = vec![];
-    for i in 0..nbytes {
+    for i in 0..len {
-        bytes.push(((n >> ((nbytes - i - 1) * 8)) & 0xFF) as u8);
+        bytes.push(((value >> ((len - i - 1) * 8)) & 0xFF) as u8);
     }
 
     bytes
@@ -63,7 +74,7 @@ fn get_int_type(value: i64) -> Vec<u8> {
     } else {
         let length = get_length_of_byte_encoding(value);
         if length < 5 {
-            varint::write(length as u64)
+            varint::write(u64::from(length))
         } else if length < 7 {
             varint::write(5)
         } else {
@@ -73,21 +84,16 @@ fn get_int_type(value: i64) -> Vec<u8> {
 }
 
 fn get_length_of_byte_encoding(value: i64) -> u8 {
-    let u =
+    let u = if value < 0 { !value } else { value };
-        if value < 0 {
-            !value
-        } else {
-            value
-        };
     if u <= 127 {
         1
-    } else if u <= 32767 {
+    } else if u <= 32_767 {
         2
-    } else if u <= 8388607 {
+    } else if u <= 8_388_607 {
         3
-    } else if u <= 2147483647 {
+    } else if u <= 2_147_483_647 {
         4
-    } else if u <= 140737488355327 {
+    } else if u <= 140_737_488_355_327 {
         6
     } else {
         8
@@ -96,7 +102,6 @@ fn get_length_of_byte_encoding(value: i64) -> u8 {
 
 #[cfg(test)]
 mod tests {
-    use crate::values::Value;
     use super::*;
 
     #[test]
@@ -147,4 +152,4 @@ mod tests {
         assert_eq!(v.datatype, vec![2]);
         assert_eq!(v.data, vec![0, 128]);
     }
-}
+}

src/varint.rs

@@ -1,16 +1,10 @@
-/// varints as implemented in SQLite
+/// varints as implemented in `SQLite`
 pub fn write(value: u64) -> Vec<u8> {
     let mut v = value;
-    if (v & ((0xff000000) << 32)) != 0 {
+    if (v & ((0xff00_0000) << 32)) == 0 {
-        let mut result = vec![0_u8; 9];
+        if v == 0 {
-        result[8] = v as u8;
+            return vec![0];
-        v >>= 8;
-        for i in (0..=7).rev() {
-            result[i] = ((v & 0x7f) | 0x80) as u8;
-            v >>= 7;
         }
-        result
-    } else {
         let mut result = Vec::new();
         while v != 0 {
             result.push(((v & 0x7f) | 0x80) as u8);
@@ -20,6 +14,15 @@ pub fn write(value: u64) -> Vec<u8> {
 
         result.reverse();
         result
+    } else {
+        let mut result = vec![0_u8; 9];
+        result[8] = v as u8;
+        v >>= 8;
+        for i in (0..=7).rev() {
+            result[i] = ((v & 0x7f) | 0x80) as u8;
+            v >>= 7;
+        }
+        result
     }
 }
 
@@ -29,13 +32,19 @@ mod tests {
 
     #[test]
     fn test() {
-        assert_eq!(vec![0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], write(0xffffffffffffffff));
+        assert_eq!(
+            vec![0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF],
+            write(0xffffffffffffffff)
+        );
     }
 
     #[test]
     fn test_write1() {
-        let a:i16 = -1;
-        println!("{}", a as u16);
         assert_eq!(vec![1], write(0x01));
     }
-}
+
+    #[test]
+    fn test_write0() {
+        assert_eq!(vec![0], write(0));
+    }
+}