import from gitlab
This commit is contained in:
Shautvast
commit
d587886f49
90 changed files with 6962 additions and 0 deletions
8
.gitignore
vendored
Normal file
8
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
*.iml
|
||||
*.db
|
||||
*.sqlite
|
||||
.idea/
|
||||
target/
|
||||
node_modules/
|
||||
dist/
|
||||
.cache/
|
||||
201
LICENSE
Normal file
201
LICENSE
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
89
README.md
Normal file
89
README.md
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
**Sqlighter**
|
||||
* Inspired by a new feature in .Net blazor (see https://www.youtube.com/watch?v=lP_qdhAHFlg&t=300s)
|
||||
* Creates a SQLite database file from any JDBC database or other tabular data
|
||||
* So instead of a rest api serving json, enables binary download of data in native SQLite format
|
||||
* So that SQLite in running in WASM (so in the browser) can be used to query the data.
|
||||
|
||||
**Why not use the sqlite code serverside for this purpose?**
|
||||
|
||||
*excellent question!*
|
||||
|
||||
But:
|
||||
1. I would have to first create and SQLite database and fill it with results and then load the database file and serve it for http requests. While this should also work, it sounds as more overhead. In Sqlighter the data stays in memory. (yes, that's a problem if the data gets reallly BIG; considering offload to file)
|
||||
2. Learning this shit is fun!
|
||||
* getting to know the fileformat. Most is nicely documented on https://sqlite.org (indeed, not everything)
|
||||
* variable length integer encoding
|
||||
* the nifty way strings and blobs are stored
|
||||
* I finally got my head around java nio (even though in the end I switched from ByteBuffer to plain byte arrays)
|
||||
* debugging this app lead me to debugging SQLite itself in xcode (really easy to set up btw). So I learnt a bit about that.
|
||||
|
||||
**Usable when:**
|
||||
* you have quite a lot of (tabular) data, that is read-only, or does not need to be (ultra) realtime.
|
||||
* and your users need to quickly apply different search criteria on it.
|
||||
* Using Sqlighter avoids server roundtrips and improves the user experience.
|
||||
* There is a utility to transfer a regular JDBC ResultSet to SQLite format, but using the API you could put any tabular data in.
|
||||
* Bear in mind that, while you, as a developer, cannot directly read the payload, like JSON allows, SQLite is available on pretty much any platform,
|
||||
and then you can leverage the power of SQL to inspect the data.
|
||||
|
||||
* other use case: you need to create an sqlite database from a file (like a CSV) in a java app. Sqlighter is much faster than jdbc+sqlite. There are no transactions, and no checks, so use at your own peril.
|
||||
* Thing to note: Sqlite is really relaxed when it comes to schema validation.
|
||||
That means that 2 records in the same table can contain values of totally different types(!). The number of values can also vary. All perfectly legal from the standpoint of Sqlighter.
|
||||
And maybe not when writing to Sqlite itself, but perfectly readable!
|
||||
|
||||
**About the name**
|
||||
* It lights up an SQLite database :)
|
||||
|
||||
**Usage**
|
||||
Creating a database is as simple as:
|
||||
```java
|
||||
DatabaseBuilder databaseBuilder = new DatabaseBuilder();
|
||||
databaseBuilder.addSchema("foo",
|
||||
"CREATE TABLE foo(bar integer,baz varchar(10))");
|
||||
|
||||
Record record = new Record(Value.of(12), Value.of("helloworld"));
|
||||
databaseBuilder.addRecord(record);
|
||||
|
||||
databaseBuilder.build().write("test.db");
|
||||
```
|
||||
* Instead of writing to file, you can also write to any `OutputStream`
|
||||
* testing with (direct) bytebuffers did not yield performance improvements for http requests (for http 1 and 2, tomcat and undertow).
|
||||
|
||||
**Be aware**
|
||||
* The schema and the actual data don't have to match! But that is how SQLite itself also works, pretty much.
|
||||
* And: 2 records in the same table can contain values of totally different types(!). The number of values can also vary. All perfectly legal from the standpoint of Sqlighter.
|
||||
And maybe not when writing to Sqlite itself, but perfectly readable!
|
||||
* unittest SchemaCreationTests is proof of this.
|
||||
|
||||
|
||||
**Current status**
|
||||
* It works for tables of any size, but probably not for indexes (may skip that feature) because you can always add them client-side(!)
|
||||
|
||||
**Performance**
|
||||
* Serverside the time that it costs is on par with [jackson](https://github.com/FasterXML/jackson):
|
||||
* that is: ~400 ms for a 9.3 Mb database (100,000 records, on my Mac m1, using jdk19/arm64)
|
||||
* versus ~430 ms using jackson (2.13.4) with the same data (payload size ~15.7Mb)
|
||||
* So it's even better, BUT in jdk11 (with rosetta) the numbers were roughly the other way around, so performing worse than jackson...
|
||||
* The response payload size is roughly 60% of the JSON. (mileage may vary, depending on the content)
|
||||
* It is comparing apples and pears though, because clientside it's way faster (and different).
|
||||
* And, it's most probably not a good idea to use the format for smaller payloads.
|
||||
* I don't know currently where the cutoff point is.
|
||||
|
||||
**DEMO**
|
||||
|
||||
|
||||
* Uses spring boot, but that is not a prerequisite for sqlighter itself.
|
||||
* see `start_api.sh` and `start_ui.sh`
|
||||
* api on http://localhost:8080
|
||||
* json on http://localhost:8080/api/customers (should provide a second ui that uses traditional json, for comparison)
|
||||
* sqlite on http://localhost:8080/api/db/customers
|
||||
* TODO should rely on content-type negotiation
|
||||
|
||||
_Frontend_
|
||||
* Now uses LIT, a web worker and OPFS and upraded to the official SQLite/WASM instead of SQL.js
|
||||
* OPFS also requires https and extra headers (see vite.config.js).
|
||||
* So far it works in Chrome and Firefox (as of V111 (I think, I only tested in 114)). Safari should but does not work (yet).
|
||||
* it has some issues still, but it supports scrolling and filtering the complete dataset.
|
||||
|
||||
**Future plans?**
|
||||
* There is now also a [rust](https://gitlab.com/sander-hautvast/sqlighte.rs) version...
|
||||
* and what about a javascript library that parses the format, so that you can render the data without sql? This could be a solution for medium sized payloads (and up) and when you just want to show the data without using the power of sql. It would be smaller than json, at least as performant and still accessible for humans with sqlite3.
|
||||
16
ci_settings.xml
Normal file
16
ci_settings.xml
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
<settings xmlns="http://maven.apache.org/SETTINGS/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.1.0 http://maven.apache.org/xsd/settings-1.1.0.xsd">
|
||||
<servers>
|
||||
<server>
|
||||
<id>gitlab-maven</id>
|
||||
<configuration>
|
||||
<httpHeaders>
|
||||
<property>
|
||||
<name>Job-Token</name>
|
||||
<value>${CI_JOB_TOKEN}</value>
|
||||
</property>
|
||||
</httpHeaders>
|
||||
</configuration>
|
||||
</server>
|
||||
</servers>
|
||||
</settings>
|
||||
6
demo/api/README.md
Normal file
6
demo/api/README.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
**Run the Demo**
|
||||
|
||||
NB. This demo requires docker
|
||||
|
||||
* run script postgresdocker.sh
|
||||
*
|
||||
76
demo/api/pom.xml
Normal file
76
demo/api/pom.xml
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<artifactId>demo</artifactId>
|
||||
<groupId>nl.sanderhautvast</groupId>
|
||||
<version>1.1</version>
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
<version>2.7.5</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>1.18.24</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>nl.sanderhautvast</groupId>
|
||||
<artifactId>sqlighter</artifactId>
|
||||
<version>1.1.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<version>2.7.5</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<version>2.7.10</version>
|
||||
<configuration>
|
||||
<mainClass>nl.sanderhautvast.sqlighter.demo.DemoApplication</mainClass>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>spring-milestones</id>
|
||||
<name>Spring Milestones</name>
|
||||
<url>https://repo.spring.io/milestone</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</repository>
|
||||
</repositories>
|
||||
<pluginRepositories>
|
||||
<pluginRepository>
|
||||
<id>spring-milestones</id>
|
||||
<name>Spring Milestones</name>
|
||||
<url>https://repo.spring.io/milestone</url>
|
||||
<snapshots>
|
||||
<enabled>false</enabled>
|
||||
</snapshots>
|
||||
</pluginRepository>
|
||||
</pluginRepositories>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package nl.sanderhautvast.sqlighter.demo;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication
|
||||
public class DemoApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(DemoApplication.class, args);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package nl.sanderhautvast.sqlighter.demo.model;
|
||||
|
||||
import lombok.*;
|
||||
|
||||
@Data
|
||||
public class Customer {
|
||||
String name;
|
||||
String email;
|
||||
String streetname;
|
||||
int housenumber;
|
||||
String city;
|
||||
String country;
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
package nl.sanderhautvast.sqlighter.demo.repository;
|
||||
|
||||
|
||||
import nl.sanderhautvast.sqlighter.Database;
|
||||
import nl.sanderhautvast.sqlighter.DatabaseBuilder;
|
||||
import nl.sanderhautvast.sqlighter.data.LtRecord;
|
||||
import nl.sanderhautvast.sqlighter.data.LtValue;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
@Repository
|
||||
public class CustomerRepository {
|
||||
|
||||
public Database getAllCustomersAsSQLite() {
|
||||
DatabaseBuilder databaseBuilder = new DatabaseBuilder();
|
||||
databaseBuilder.addSchema("customers",
|
||||
"create table customers (name varchar(100), email varchar(100), streetname varchar(100), housenumber integer, city varchar(100), country varchar(100))");
|
||||
|
||||
final RandomStuffGenerator generator = new RandomStuffGenerator();
|
||||
long rowid = 1;
|
||||
|
||||
for (int i = 0; i < 100_000; i++) {
|
||||
LtRecord record = new LtRecord(rowid++);
|
||||
record.addValues();
|
||||
|
||||
String firstName = generator.generateFirstName();
|
||||
String lastName = generator.generateLastName();
|
||||
record.addValues(LtValue.of(firstName + " " + lastName),
|
||||
LtValue.of(firstName + "." + lastName + "@icemail.com"),
|
||||
LtValue.of(generator.generateStreetName()),
|
||||
LtValue.of(generator.generateSomeNumber()),
|
||||
LtValue.of(generator.generateSomeCityInIceland()),
|
||||
LtValue.of(generator.generateIceland()));
|
||||
|
||||
databaseBuilder.addRecord(record);
|
||||
}
|
||||
return databaseBuilder.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
package nl.sanderhautvast.sqlighter.demo.repository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
public class RandomStuffGenerator {
|
||||
|
||||
private final List<String> firstNameParts = List.of("sa", "ka", "zo", "ja", "za", "ka", "po", "ji", "ne", "si", "wi", "ha", "ut", "va", "no", "bo"
|
||||
, "jo", "fe", "gu");
|
||||
|
||||
private final List<String> lastNameParts = List.of("fin", "wil", "cat", "loc", "der", "ter", "asp", "pen", "ill", "raf", "gut", "dax", "yin");
|
||||
private final List<String> cities = List.of("Reykjavík", "Kópavogur", "Hafnarfjörður", "Akureyri", "Reykjanesbær", "Garðabær", "Mosfellsbær", "Selfoss", "Akranes", "Seltjarnarnes", "Vestmannaeyjar", "Grindavík", "Ísafjörður", "Álftanes", "Sauðárkrókur", "Hveragerði", "Egilsstaðir", "Húsavík", "Borgarnes", "Sandgerði", "Höfn", "Þorlákshöfn", "Garður", "Neskaupstaður", "Dalvík", "Reyðarfjörður", "Siglufjörður", "Vogar", "Stykkishólmur", "Eskifjörður", "Ólafsvík", "Hvolsvöllur", "Bolungarvík", "Hella", "Grundarfjörður", "Blönduós", "Ólafsfjörður", "Fáskrúðsfjörður", "Patreksfjörður", "Seyðisfjörður", "Grundarhverfi", "Hvammstangi", "Stokkseyri", "Eyrarbakki", "Vopnafjörður", "Skagaströnd", "Flúðir", "Vík", "Fellabær", "Hellissandur", "Djúpivogur", "Þórshöfn", "Svalbarðseyri", "Hólmavík", "Grenivík", "Hvanneyri", "Þingeyri", "Búðardalur", "Reykholt", "Hrafnagil", "Suðureyri", "Tálknafjörður", "Bíldudalur", "Mosfellsdalur", "Hnífsdalur", "Reykjahlíð", "Laugarvatn", "Raufarhöfn", "Stöðvarfjörður", "Bifröst", "Flateyri", "Kirkjubæjarklaustur", "Súðavík", "Hrísey", "Hofsós", "Breiðdalsvík", "Rif", "Reykhólar", "Varmahlíð", "Kópasker", "Laugarás", "Borg", "Hauganes", "Hafnir", "Laugar", "Melahverfi", "Tjarnabyggð", "Árskógssandur", "Lónsbakki", "Hólar", "Nesjahverfi", "Sólheimar", "Brúnahlíð", "Drangsnes", "Borgarfjörður eystri", "Árbæjarhverfi", "Brautarholt", "Rauðalækur", "Bakkafjörður", "Innnes", "Grímsey", "Þykkvabær", "Laugarbakki", "Reykholt", "Árnes", "Kristnes", "Kleppjárnsreykir");
|
||||
private final Random random = new Random();
|
||||
|
||||
public String generateFirstName() {
|
||||
return generateName(firstNameParts);
|
||||
}
|
||||
|
||||
public String generateLastName() {
|
||||
return generateName(lastNameParts);
|
||||
}
|
||||
|
||||
public String generateStreetName() {
|
||||
StringBuilder name = new StringBuilder();
|
||||
int nLastNameParts = random.nextInt(5) + 1;
|
||||
for (int i = 0; i < nLastNameParts; i++) {
|
||||
name.append(firstNameParts.get(random.nextInt(firstNameParts.size())));
|
||||
name.append(lastNameParts.get(random.nextInt(lastNameParts.size())));
|
||||
}
|
||||
name.append("götu");
|
||||
return name.toString();
|
||||
}
|
||||
|
||||
public int generateSomeNumber() {
|
||||
return random.nextInt(1000);
|
||||
}
|
||||
|
||||
public String generateSomeCityInIceland() {
|
||||
return cities.get(random.nextInt(cities.size()));
|
||||
}
|
||||
|
||||
public String generateIceland() {
|
||||
return "Iceland"; // meant to be humorous
|
||||
}
|
||||
|
||||
private String generateName(List<String> parts) {
|
||||
StringBuilder name = new StringBuilder();
|
||||
int size = random.nextInt(2) + 2;
|
||||
for (int i = 0; i < size; i++) {
|
||||
name.append(parts.get(random.nextInt(parts.size())));
|
||||
}
|
||||
return name.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
package nl.sanderhautvast.sqlighter.demo.rest;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.Database;
|
||||
import nl.sanderhautvast.sqlighter.demo.repository.CustomerRepository;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import java.io.IOException;
|
||||
|
||||
@RestController
|
||||
public class DemoRestApi {
|
||||
|
||||
private final CustomerRepository customerRepository;
|
||||
|
||||
@Autowired
|
||||
public DemoRestApi(CustomerRepository customerRepository) {
|
||||
this.customerRepository = customerRepository;
|
||||
}
|
||||
|
||||
@GetMapping(value = "/api/db/customers", produces = "application/octet-stream")
|
||||
public void getDataAsSqliteFile(HttpServletResponse response) throws IOException {
|
||||
Database customers = customerRepository.getAllCustomersAsSQLite();
|
||||
customers.write(response.getOutputStream());
|
||||
}
|
||||
}
|
||||
7
demo/api/src/main/resources/application.properties
Normal file
7
demo/api/src/main/resources/application.properties
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# compression and http2
|
||||
server.compression.enabled=true
|
||||
server.compression.mime-types=application/octet-stream
|
||||
server.compression.min-response-size=1024
|
||||
#server.http2.enabled=true
|
||||
# try it with, for instance:
|
||||
# curl http://localhost:8080/api/db/customers --output customers --compressed -v --http2
|
||||
3
demo/start_api.sh
Normal file
3
demo/start_api.sh
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
mvn -f api/pom.xml -DskipTests clean spring-boot:run
|
||||
|
||||
|
||||
7
demo/start_ui.sh
Normal file
7
demo/start_ui.sh
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
cd ui
|
||||
npm run dev
|
||||
|
||||
# https://localhost:5173/
|
||||
# NB. this demo does not run in firefox or safari, because it does not support OPFS yet!
|
||||
|
||||
# NB2. TLS is enabled by signed dummy cert for localhost
|
||||
26
demo/ui/.cert/cert.pem
Normal file
26
demo/ui/.cert/cert.pem
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
-----BEGIN CERTIFICATE-----
|
||||
MIIEdzCCAt+gAwIBAgIRAMyMIvlBA6nant7D7dumdNQwDQYJKoZIhvcNAQELBQAw
|
||||
gaUxHjAcBgNVBAoTFW1rY2VydCBkZXZlbG9wbWVudCBDQTE9MDsGA1UECww0U2hh
|
||||
dXR2YXN0QG1hY2Jvb2stcHJvLnR3aW5rbGVzcGFyayAoU2FuZGVyIEhhdXR2YXN0
|
||||
KTFEMEIGA1UEAww7bWtjZXJ0IFNoYXV0dmFzdEBtYWNib29rLXByby50d2lua2xl
|
||||
c3BhcmsgKFNhbmRlciBIYXV0dmFzdCkwHhcNMjMwMzEzMTI0MzIyWhcNMjUwNjEz
|
||||
MTE0MzIyWjBoMScwJQYDVQQKEx5ta2NlcnQgZGV2ZWxvcG1lbnQgY2VydGlmaWNh
|
||||
dGUxPTA7BgNVBAsMNFNoYXV0dmFzdEBtYWNib29rLXByby50d2lua2xlc3Bhcmsg
|
||||
KFNhbmRlciBIYXV0dmFzdCkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
|
||||
AQDO3UIb3OjeSjRV+C9FQ00K16oYY5H41GjtbCav3ZChf0LCJVr1SOAyWtOpZlgP
|
||||
5GxGVjaXvD3MSYog8SpMOMHd6+B7dp02sSJPV3RdcCtSKo1WgXGkSZ59neoSRSzF
|
||||
BwR/3sGyPyXsl4BFR+moQfORuwLh2FIEAdQFGRbLpRfunjvVv8vvbFBtyFSMndMV
|
||||
8jvs3k3MYiwqBVzaIuT/GZ+AVUSxSzHtKRO3+2er/6xtEQTZ03Icjb0vMBGfzBBK
|
||||
zBcUZN4wXC2chSFI1Ptixp/yaec1dkZ2JvaGoznhmexvgWfBpvYPjmYM/bHm0BPm
|
||||
9bsxGTft4vecbx4VGRT7PR5pAgMBAAGjXjBcMA4GA1UdDwEB/wQEAwIFoDATBgNV
|
||||
HSUEDDAKBggrBgEFBQcDATAfBgNVHSMEGDAWgBT8C0UzAyzvp3PMf5SUIU76Xcqx
|
||||
6TAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQADggGBAHXoL9Zi
|
||||
wq26HeOA5RA+7QRHJElCEz2Epl3nSURXggKUAtL8svzOPXtNLLBgKUjUErbT6rIy
|
||||
PrkF9ZBwnbbMFpy/e//sJPNI/XfG4L6+ZIP/bmEYBc/GAkS3ybeJiNS0EK5M1Qm7
|
||||
z5g4wwOIg+eCOYu4e9chOa3p+Zoopdcp/jh6C7AF4+MxuQmHmA09e042TgNsZhph
|
||||
SFcii10+vI+u0EH9N106jOGDF/YyV4i4oVdigds/YF5tGVf9QAUXl71pwE7upyz9
|
||||
JWXBTFGskvfNhW+6UNKPfe1KEWySd4BOQH4Vceqv8Om1jnBbPJSoI/vRJ0OO9cLH
|
||||
PSUdyb9UCfc1ajk/79PzzWdnaZGKKRDwesnbm6v+P47nDZDIOfvOXgMpOFos1Ozk
|
||||
nT8388vc0q0P7/VpKz5eQHPssWQgAd9aN74hLVz1Y7oGzVRef6A/pcGaJBbqgH24
|
||||
WWEPJF24qFWq9idBfq8Jd//T77x0BL5s2JHZlGNFJ7VRWnZZ5a8FaKgcxA==
|
||||
-----END CERTIFICATE-----
|
||||
28
demo/ui/.cert/key.pem
Normal file
28
demo/ui/.cert/key.pem
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDO3UIb3OjeSjRV
|
||||
+C9FQ00K16oYY5H41GjtbCav3ZChf0LCJVr1SOAyWtOpZlgP5GxGVjaXvD3MSYog
|
||||
8SpMOMHd6+B7dp02sSJPV3RdcCtSKo1WgXGkSZ59neoSRSzFBwR/3sGyPyXsl4BF
|
||||
R+moQfORuwLh2FIEAdQFGRbLpRfunjvVv8vvbFBtyFSMndMV8jvs3k3MYiwqBVza
|
||||
IuT/GZ+AVUSxSzHtKRO3+2er/6xtEQTZ03Icjb0vMBGfzBBKzBcUZN4wXC2chSFI
|
||||
1Ptixp/yaec1dkZ2JvaGoznhmexvgWfBpvYPjmYM/bHm0BPm9bsxGTft4vecbx4V
|
||||
GRT7PR5pAgMBAAECggEAALHFcwp+zaA6JL+8iOL88zheHc7XwpoT8BgY2SZJQgGH
|
||||
W9d8Qq6H8iNT6pGpYgRGSMYpUMEVBAaLLCkjFzZpkDEmg8OWK4TUtKd6gIXL3269
|
||||
Irk0hHSqz4kkLAOHbvHjFfrNc6e6nNy1VYY0RphSFP8nt1JelBzddVbsOKrJNcfY
|
||||
p7IQtyeS//H+CEQ66gvVytUbtNu3Zzn7oXmw7PS9+UkxrDoyu+P6joG4LE3B3QHS
|
||||
LUXjersiDTuTKeWBSDd5PNSKAYX/FHdt3yoo98etDcZESpRnmJ6ANljREOwdbhaH
|
||||
0vN8kG3GMvK+5eCs2YxUHCcukottpSc9AhDx8/DAAQKBgQDcnOyNG/Dnntid2pc+
|
||||
+u/lFPnEsYWIsEtsMGEdKb8LfwGAiptZtUYtdH1hYOYXfhJxGY3tJxImIE1WeFq9
|
||||
dH1pzMK4NRCUtm6+Tsex6lzVS9Cao8t3PvrIEo+ZgC8S2R6MKqAp1WPRhCzYH/1s
|
||||
HzBBVUPIo+PgqiRau5mD7MgkaQKBgQDwC8TNspAs277d9TYTyK6+yzVc7PzqhhO0
|
||||
yMzzTG5xEEJcGN+ZR1kR6rwR5xu29AQuZ8JRCbcwO4nVXoTlx3bgIFRgFzhG59KA
|
||||
ik5L3hBikB3gh4xZqRiYhX9dMX9iqQzZ3aWdx7x3xDnBx1HPFYcJltbq8+LxCIlp
|
||||
ZTBqyiXqAQKBgBArA/8XdCFVd+Sht4HrHBe64M80f2fUG8LzDLr5a0Hpbe/AuL/r
|
||||
VBhSuDmhw1snZyyYxdkCiwb2SRS0P1oxJlvRoNelM/DiKd2Sonn8hg8vvjsHFAtK
|
||||
N1DgY1vJlCmade2p1hEazXT2bd7tAUKiSoQPPqd+s12suntX0lljygs5AoGBAOCN
|
||||
cdYmsz5zdlJ2P9c0BVwQBmRegZ32PNsCeM8kcbAs0JcM4aammsjq+HIa6s8z5/Ft
|
||||
ONbMKuTg3WiPWe0FscuqEqQtNIUH+eArAWFxY4yAWqKeyolZaNvNDj8kvZCSqaXo
|
||||
9TPrFABJvOnsRjhdYAx1Yak66tIl9T113lwXrG4BAoGBAMVQbmaO4zEkUWNFfxd1
|
||||
WQTRtqQqdz4QBAmEf0HmrXGNKhqZNppMO2YndZzdxV2aEHPKgOe0SS3oyydxjAkF
|
||||
MRJLPwIpMGMKoknC0lfT7jw2Ma6a2L4Lc6Ds9NCGTOJc8jpW6f6VSV6mYHxd7nOp
|
||||
NRF8O60CKfz1144nBLmdiBUZ
|
||||
-----END PRIVATE KEY-----
|
||||
23
demo/ui/index.html
Normal file
23
demo/ui/index.html
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<title>Sqlighter demo</title>
|
||||
<meta charset="UTF-8"/>
|
||||
</head>
|
||||
|
||||
<body id="body">
|
||||
<div class="header-row">
|
||||
<div class="header"><img id="s-name" src="src/sort.svg"/>name<a class ="filter" id="name">≡</a></div>
|
||||
<div class="header"><img id="s-email" src="src/sort.svg"/>email<a class="filter" id="email">≡</a></div>
|
||||
<div class="header"><img id="s-streetname" src="src/sort.svg"/>street<a class="filter" id="streetname">≡</a></div>
|
||||
<div class="header"><img id="s-housenumber" src="src/sort.svg"/>number<a class="filter" id="housenumber">≡</a></div>
|
||||
<div class="header"><img id="s-city" src="src/sort.svg"/>city<a class="filter" id="city">≡</a></div>
|
||||
<div class="header"><img id="s-country" src="src/sort.svg"/>country<a class="filter" id="country">≡</a></div>
|
||||
</div>
|
||||
|
||||
|
||||
<virtual-scroller></virtual-scroller>
|
||||
|
||||
<script type="module" src="src/app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
2360
demo/ui/package-lock.json
generated
Normal file
2360
demo/ui/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
20
demo/ui/package.json
Normal file
20
demo/ui/package.json
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
{
|
||||
"name": "lit-virtual-scroll-basic",
|
||||
"type": "module",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"scripts": {
|
||||
"start": "npm run dev",
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"serve": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"lit": "^2.6.1",
|
||||
"sqlite-wasm-esm": "^0.0.30"
|
||||
},
|
||||
"devDependencies": {
|
||||
"vite": "^4.1.4"
|
||||
},
|
||||
"keywords": ["sqlite", "lit"]
|
||||
}
|
||||
390
demo/ui/pnpm-lock.yaml
generated
Normal file
390
demo/ui/pnpm-lock.yaml
generated
Normal file
|
|
@ -0,0 +1,390 @@
|
|||
lockfileVersion: 5.4
|
||||
|
||||
specifiers:
|
||||
lit: ^2.6.1
|
||||
sqlite-wasm-esm: ^0.0.30
|
||||
vite: ^4.1.4
|
||||
|
||||
dependencies:
|
||||
lit: 2.6.1
|
||||
sqlite-wasm-esm: 0.0.30
|
||||
|
||||
devDependencies:
|
||||
vite: 4.1.4
|
||||
|
||||
packages:
|
||||
|
||||
/@esbuild/android-arm/0.16.17:
|
||||
resolution: {integrity: sha512-N9x1CMXVhtWEAMS7pNNONyA14f71VPQN9Cnavj1XQh6T7bskqiLLrSca4O0Vr8Wdcga943eThxnVp3JLnBMYtw==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm]
|
||||
os: [android]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/android-arm64/0.16.17:
|
||||
resolution: {integrity: sha512-MIGl6p5sc3RDTLLkYL1MyL8BMRN4tLMRCn+yRJJmEDvYZ2M7tmAf80hx1kbNEUX2KJ50RRtxZ4JHLvCfuB6kBg==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm64]
|
||||
os: [android]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/android-x64/0.16.17:
|
||||
resolution: {integrity: sha512-a3kTv3m0Ghh4z1DaFEuEDfz3OLONKuFvI4Xqczqx4BqLyuFaFkuaG4j2MtA6fuWEFeC5x9IvqnX7drmRq/fyAQ==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [android]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/darwin-arm64/0.16.17:
|
||||
resolution: {integrity: sha512-/2agbUEfmxWHi9ARTX6OQ/KgXnOWfsNlTeLcoV7HSuSTv63E4DqtAc+2XqGw1KHxKMHGZgbVCZge7HXWX9Vn+w==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm64]
|
||||
os: [darwin]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/darwin-x64/0.16.17:
|
||||
resolution: {integrity: sha512-2By45OBHulkd9Svy5IOCZt376Aa2oOkiE9QWUK9fe6Tb+WDr8hXL3dpqi+DeLiMed8tVXspzsTAvd0jUl96wmg==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [darwin]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/freebsd-arm64/0.16.17:
|
||||
resolution: {integrity: sha512-mt+cxZe1tVx489VTb4mBAOo2aKSnJ33L9fr25JXpqQqzbUIw/yzIzi+NHwAXK2qYV1lEFp4OoVeThGjUbmWmdw==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm64]
|
||||
os: [freebsd]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/freebsd-x64/0.16.17:
|
||||
resolution: {integrity: sha512-8ScTdNJl5idAKjH8zGAsN7RuWcyHG3BAvMNpKOBaqqR7EbUhhVHOqXRdL7oZvz8WNHL2pr5+eIT5c65kA6NHug==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [freebsd]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-arm/0.16.17:
|
||||
resolution: {integrity: sha512-iihzrWbD4gIT7j3caMzKb/RsFFHCwqqbrbH9SqUSRrdXkXaygSZCZg1FybsZz57Ju7N/SHEgPyaR0LZ8Zbe9gQ==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-arm64/0.16.17:
|
||||
resolution: {integrity: sha512-7S8gJnSlqKGVJunnMCrXHU9Q8Q/tQIxk/xL8BqAP64wchPCTzuM6W3Ra8cIa1HIflAvDnNOt2jaL17vaW+1V0g==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-ia32/0.16.17:
|
||||
resolution: {integrity: sha512-kiX69+wcPAdgl3Lonh1VI7MBr16nktEvOfViszBSxygRQqSpzv7BffMKRPMFwzeJGPxcio0pdD3kYQGpqQ2SSg==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [ia32]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-loong64/0.16.17:
|
||||
resolution: {integrity: sha512-dTzNnQwembNDhd654cA4QhbS9uDdXC3TKqMJjgOWsC0yNCbpzfWoXdZvp0mY7HU6nzk5E0zpRGGx3qoQg8T2DQ==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [loong64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-mips64el/0.16.17:
|
||||
resolution: {integrity: sha512-ezbDkp2nDl0PfIUn0CsQ30kxfcLTlcx4Foz2kYv8qdC6ia2oX5Q3E/8m6lq84Dj/6b0FrkgD582fJMIfHhJfSw==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [mips64el]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-ppc64/0.16.17:
|
||||
resolution: {integrity: sha512-dzS678gYD1lJsW73zrFhDApLVdM3cUF2MvAa1D8K8KtcSKdLBPP4zZSLy6LFZ0jYqQdQ29bjAHJDgz0rVbLB3g==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [ppc64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-riscv64/0.16.17:
|
||||
resolution: {integrity: sha512-ylNlVsxuFjZK8DQtNUwiMskh6nT0vI7kYl/4fZgV1llP5d6+HIeL/vmmm3jpuoo8+NuXjQVZxmKuhDApK0/cKw==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [riscv64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-s390x/0.16.17:
|
||||
resolution: {integrity: sha512-gzy7nUTO4UA4oZ2wAMXPNBGTzZFP7mss3aKR2hH+/4UUkCOyqmjXiKpzGrY2TlEUhbbejzXVKKGazYcQTZWA/w==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [s390x]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/linux-x64/0.16.17:
|
||||
resolution: {integrity: sha512-mdPjPxfnmoqhgpiEArqi4egmBAMYvaObgn4poorpUaqmvzzbvqbowRllQ+ZgzGVMGKaPkqUmPDOOFQRUFDmeUw==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/netbsd-x64/0.16.17:
|
||||
resolution: {integrity: sha512-/PzmzD/zyAeTUsduZa32bn0ORug+Jd1EGGAUJvqfeixoEISYpGnAezN6lnJoskauoai0Jrs+XSyvDhppCPoKOA==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [netbsd]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/openbsd-x64/0.16.17:
|
||||
resolution: {integrity: sha512-2yaWJhvxGEz2RiftSk0UObqJa/b+rIAjnODJgv2GbGGpRwAfpgzyrg1WLK8rqA24mfZa9GvpjLcBBg8JHkoodg==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [openbsd]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/sunos-x64/0.16.17:
|
||||
resolution: {integrity: sha512-xtVUiev38tN0R3g8VhRfN7Zl42YCJvyBhRKw1RJjwE1d2emWTVToPLNEQj/5Qxc6lVFATDiy6LjVHYhIPrLxzw==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [sunos]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/win32-arm64/0.16.17:
|
||||
resolution: {integrity: sha512-ga8+JqBDHY4b6fQAmOgtJJue36scANy4l/rL97W+0wYmijhxKetzZdKOJI7olaBaMhWt8Pac2McJdZLxXWUEQw==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [arm64]
|
||||
os: [win32]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/win32-ia32/0.16.17:
|
||||
resolution: {integrity: sha512-WnsKaf46uSSF/sZhwnqE4L/F89AYNMiD4YtEcYekBt9Q7nj0DiId2XH2Ng2PHM54qi5oPrQ8luuzGszqi/veig==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [ia32]
|
||||
os: [win32]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@esbuild/win32-x64/0.16.17:
|
||||
resolution: {integrity: sha512-y+EHuSchhL7FjHgvQL/0fnnFmO4T1bhvWANX6gcnqTjtnKWbTvUMCpGnv2+t+31d7RzyEAYAd4u2fnIhHL6N/Q==}
|
||||
engines: {node: '>=12'}
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/@lit-labs/ssr-dom-shim/1.0.0:
|
||||
resolution: {integrity: sha512-ic93MBXfApIFTrup4a70M/+ddD8xdt2zxxj9sRwHQzhS9ag/syqkD8JPdTXsc1gUy2K8TTirhlCqyTEM/sifNw==}
|
||||
dev: false
|
||||
|
||||
/@lit/reactive-element/1.6.1:
|
||||
resolution: {integrity: sha512-va15kYZr7KZNNPZdxONGQzpUr+4sxVu7V/VG7a8mRfPPXUyhEYj5RzXCQmGrlP3tAh0L3HHm5AjBMFYRqlM9SA==}
|
||||
dependencies:
|
||||
'@lit-labs/ssr-dom-shim': 1.0.0
|
||||
dev: false
|
||||
|
||||
/@types/trusted-types/2.0.3:
|
||||
resolution: {integrity: sha512-NfQ4gyz38SL8sDNrSixxU2Os1a5xcdFxipAFxYEuLUlvU2uDwS4NUpsImcf1//SlWItCVMMLiylsxbmNMToV/g==}
|
||||
dev: false
|
||||
|
||||
/esbuild/0.16.17:
|
||||
resolution: {integrity: sha512-G8LEkV0XzDMNwXKgM0Jwu3nY3lSTwSGY6XbxM9cr9+s0T/qSV1q1JVPBGzm3dcjhCic9+emZDmMffkwgPeOeLg==}
|
||||
engines: {node: '>=12'}
|
||||
hasBin: true
|
||||
requiresBuild: true
|
||||
optionalDependencies:
|
||||
'@esbuild/android-arm': 0.16.17
|
||||
'@esbuild/android-arm64': 0.16.17
|
||||
'@esbuild/android-x64': 0.16.17
|
||||
'@esbuild/darwin-arm64': 0.16.17
|
||||
'@esbuild/darwin-x64': 0.16.17
|
||||
'@esbuild/freebsd-arm64': 0.16.17
|
||||
'@esbuild/freebsd-x64': 0.16.17
|
||||
'@esbuild/linux-arm': 0.16.17
|
||||
'@esbuild/linux-arm64': 0.16.17
|
||||
'@esbuild/linux-ia32': 0.16.17
|
||||
'@esbuild/linux-loong64': 0.16.17
|
||||
'@esbuild/linux-mips64el': 0.16.17
|
||||
'@esbuild/linux-ppc64': 0.16.17
|
||||
'@esbuild/linux-riscv64': 0.16.17
|
||||
'@esbuild/linux-s390x': 0.16.17
|
||||
'@esbuild/linux-x64': 0.16.17
|
||||
'@esbuild/netbsd-x64': 0.16.17
|
||||
'@esbuild/openbsd-x64': 0.16.17
|
||||
'@esbuild/sunos-x64': 0.16.17
|
||||
'@esbuild/win32-arm64': 0.16.17
|
||||
'@esbuild/win32-ia32': 0.16.17
|
||||
'@esbuild/win32-x64': 0.16.17
|
||||
dev: true
|
||||
|
||||
/fsevents/2.3.2:
|
||||
resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==}
|
||||
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
|
||||
os: [darwin]
|
||||
requiresBuild: true
|
||||
dev: true
|
||||
optional: true
|
||||
|
||||
/function-bind/1.1.1:
|
||||
resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==}
|
||||
dev: true
|
||||
|
||||
/has/1.0.3:
|
||||
resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==}
|
||||
engines: {node: '>= 0.4.0'}
|
||||
dependencies:
|
||||
function-bind: 1.1.1
|
||||
dev: true
|
||||
|
||||
/is-core-module/2.11.0:
|
||||
resolution: {integrity: sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==}
|
||||
dependencies:
|
||||
has: 1.0.3
|
||||
dev: true
|
||||
|
||||
/lit-element/3.2.2:
|
||||
resolution: {integrity: sha512-6ZgxBR9KNroqKb6+htkyBwD90XGRiqKDHVrW/Eh0EZ+l+iC+u+v+w3/BA5NGi4nizAVHGYvQBHUDuSmLjPp7NQ==}
|
||||
dependencies:
|
||||
'@lit/reactive-element': 1.6.1
|
||||
lit-html: 2.6.1
|
||||
dev: false
|
||||
|
||||
/lit-html/2.6.1:
|
||||
resolution: {integrity: sha512-Z3iw+E+3KKFn9t2YKNjsXNEu/LRLI98mtH/C6lnFg7kvaqPIzPn124Yd4eT/43lyqrejpc5Wb6BHq3fdv4S8Rw==}
|
||||
dependencies:
|
||||
'@types/trusted-types': 2.0.3
|
||||
dev: false
|
||||
|
||||
/lit/2.6.1:
|
||||
resolution: {integrity: sha512-DT87LD64f8acR7uVp7kZfhLRrHkfC/N4BVzAtnw9Yg8087mbBJ//qedwdwX0kzDbxgPccWRW6mFwGbRQIxy0pw==}
|
||||
dependencies:
|
||||
'@lit/reactive-element': 1.6.1
|
||||
lit-element: 3.2.2
|
||||
lit-html: 2.6.1
|
||||
dev: false
|
||||
|
||||
/nanoid/3.3.4:
|
||||
resolution: {integrity: sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==}
|
||||
engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
|
||||
hasBin: true
|
||||
dev: true
|
||||
|
||||
/path-parse/1.0.7:
|
||||
resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==}
|
||||
dev: true
|
||||
|
||||
/picocolors/1.0.0:
|
||||
resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==}
|
||||
dev: true
|
||||
|
||||
/postcss/8.4.21:
|
||||
resolution: {integrity: sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg==}
|
||||
engines: {node: ^10 || ^12 || >=14}
|
||||
dependencies:
|
||||
nanoid: 3.3.4
|
||||
picocolors: 1.0.0
|
||||
source-map-js: 1.0.2
|
||||
dev: true
|
||||
|
||||
/resolve/1.22.1:
|
||||
resolution: {integrity: sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==}
|
||||
hasBin: true
|
||||
dependencies:
|
||||
is-core-module: 2.11.0
|
||||
path-parse: 1.0.7
|
||||
supports-preserve-symlinks-flag: 1.0.0
|
||||
dev: true
|
||||
|
||||
/rollup/3.19.1:
|
||||
resolution: {integrity: sha512-lAbrdN7neYCg/8WaoWn/ckzCtz+jr70GFfYdlf50OF7387HTg+wiuiqJRFYawwSPpqfqDNYqK7smY/ks2iAudg==}
|
||||
engines: {node: '>=14.18.0', npm: '>=8.0.0'}
|
||||
hasBin: true
|
||||
optionalDependencies:
|
||||
fsevents: 2.3.2
|
||||
dev: true
|
||||
|
||||
/source-map-js/1.0.2:
|
||||
resolution: {integrity: sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
dev: true
|
||||
|
||||
/sqlite-wasm-esm/0.0.30:
|
||||
resolution: {integrity: sha512-rLl+STKLfGXyBcpQlH6uEMMh76YXixY3s+qDEMzIiMMsyN7iXLmo4Mk1Su/6GoJFprSWP+cgOCWQsAbLELCEPg==}
|
||||
dev: false
|
||||
|
||||
/supports-preserve-symlinks-flag/1.0.0:
|
||||
resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==}
|
||||
engines: {node: '>= 0.4'}
|
||||
dev: true
|
||||
|
||||
/vite/4.1.4:
|
||||
resolution: {integrity: sha512-3knk/HsbSTKEin43zHu7jTwYWv81f8kgAL99G5NWBcA1LKvtvcVAC4JjBH1arBunO9kQka+1oGbrMKOjk4ZrBg==}
|
||||
engines: {node: ^14.18.0 || >=16.0.0}
|
||||
hasBin: true
|
||||
peerDependencies:
|
||||
'@types/node': '>= 14'
|
||||
less: '*'
|
||||
sass: '*'
|
||||
stylus: '*'
|
||||
sugarss: '*'
|
||||
terser: ^5.4.0
|
||||
peerDependenciesMeta:
|
||||
'@types/node':
|
||||
optional: true
|
||||
less:
|
||||
optional: true
|
||||
sass:
|
||||
optional: true
|
||||
stylus:
|
||||
optional: true
|
||||
sugarss:
|
||||
optional: true
|
||||
terser:
|
||||
optional: true
|
||||
dependencies:
|
||||
esbuild: 0.16.17
|
||||
postcss: 8.4.21
|
||||
resolve: 1.22.1
|
||||
rollup: 3.19.1
|
||||
optionalDependencies:
|
||||
fsevents: 2.3.2
|
||||
dev: true
|
||||
206
demo/ui/src/app.js
Normal file
206
demo/ui/src/app.js
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
import {LitElement, html} from 'lit';
|
||||
import './item.js';
|
||||
import './virtual-scroll';
|
||||
import './styles.css';
|
||||
|
||||
const columns = ['name', 'email', 'streetname', 'housenumber', 'city', 'country'];
|
||||
|
||||
class App extends LitElement {
|
||||
|
||||
static get properties() {
|
||||
return {
|
||||
data: {type: Array},
|
||||
dbWorker: {},
|
||||
filters: {type: Object},
|
||||
where: {},
|
||||
sort: {type: Object}
|
||||
}
|
||||
}
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.filters = {
|
||||
name: {visible: "hidden", value: ""},
|
||||
email: {visible: "hidden", value: ""},
|
||||
streetname: {visible: "hidden", value: ""},
|
||||
housenumber: {visible: "hidden", value: ""},
|
||||
city: {visible: "hidden", value: ""},
|
||||
country: {visible: "hidden", value: ""},
|
||||
}
|
||||
this.where = "";
|
||||
this.data = [];
|
||||
this.sort = {
|
||||
prevColumn: undefined,
|
||||
column: undefined,
|
||||
order: undefined
|
||||
}
|
||||
|
||||
// sqlite needs a worker thread
|
||||
// there is also a promise based sqlite wrapper, but the esm wrapper used here does not support that
|
||||
this.dbWorker = new Worker(new URL('./database.js', import.meta.url), {
|
||||
type: 'module',
|
||||
});
|
||||
|
||||
// listen for events from the database worker
|
||||
this.dbWorker.onmessage = (e) => {
|
||||
if (e.data === "db ready") {
|
||||
this.dbWorker.postMessage("SELECT rowid, name, email, streetname, housenumber, city, country FROM customers");
|
||||
} else {
|
||||
if (e.data) {
|
||||
this.data = e.data;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// listen for user input, execute the updated query
|
||||
this.addEventListener('filterupdate-event', e => {
|
||||
this.where = e.detail.message;
|
||||
this.updateQuery();
|
||||
});
|
||||
|
||||
document.getElementById("body").addEventListener("keyup", e => {
|
||||
if (e.code === 'Escape') {
|
||||
columns.forEach(c => this.filters[c].visible = "hidden");
|
||||
}
|
||||
this.requestUpdate();
|
||||
});
|
||||
}
|
||||
|
||||
updateQuery() {
|
||||
let query = "SELECT rowid, name, email, streetname, housenumber, city, country FROM customers";
|
||||
if (this.where.length > 0) {
|
||||
query += ` WHERE ${this.where}`;
|
||||
}
|
||||
if (this.sort.column && this.sort.order) {
|
||||
query += ` ORDER BY ${this.sort.column} ${this.sort.order}`
|
||||
}
|
||||
this.dbWorker.postMessage(query);
|
||||
}
|
||||
|
||||
firstUpdated() {
|
||||
columns.forEach(c => {
|
||||
document.getElementById(c).addEventListener("click", e => {
|
||||
this.filters[e.target.id].visible = "visible";
|
||||
this.requestUpdate();
|
||||
setTimeout(() => this.shadowRoot.getElementById(`i-${e.target.id}`).focus(), 10);
|
||||
});
|
||||
|
||||
document.getElementById(`s-${c}`).addEventListener("click", e => {
|
||||
this.sort.prevColumn = this.sort.column;
|
||||
this.sort.column = e.target.id.substring(2);
|
||||
if (this.sort.prevColumn !== this.sort.column){
|
||||
this.sort.order = undefined;
|
||||
}
|
||||
|
||||
if (!this.sort.order) {
|
||||
this.sort.order = "ASC";
|
||||
} else if (this.sort.order === 'ASC') {
|
||||
this.sort.order = "DESC";
|
||||
} else {
|
||||
this.sort.order = undefined;
|
||||
}
|
||||
this.updateQuery();
|
||||
});
|
||||
|
||||
const element = this.shadowRoot.getElementById(`f-${c}`);
|
||||
element.addEventListener("keyup", e => {
|
||||
if (e.code === 'Escape' || e.code === 'Enter') {
|
||||
this.filters[e.target.id.substring(2)].visible = "hidden";
|
||||
this.requestUpdate();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
element.addEventListener("focusout", e => {
|
||||
this.filters[e.target.id.substring(2)].visible = "hidden";
|
||||
this.requestUpdate();
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
render() {
|
||||
return html`
|
||||
<style>
|
||||
:host .filter {
|
||||
z-index: 1;
|
||||
border: solid #888 2px;
|
||||
border-radius: 15px;
|
||||
background: white;
|
||||
position: absolute;
|
||||
top: 2%;
|
||||
width: 10em;
|
||||
}
|
||||
|
||||
input {
|
||||
margin: 5px;
|
||||
z-index: 2;
|
||||
border: none;
|
||||
width: auto;
|
||||
}
|
||||
|
||||
input:focus {
|
||||
outline: none !important;
|
||||
|
||||
}
|
||||
</style>
|
||||
|
||||
<div>
|
||||
<div class="filter" id="f-name"
|
||||
style="left: 2%;visibility: ${this.filters.name.visible}">
|
||||
<input id="i-name" placeholder="name" .value=${this.filters.name.value}
|
||||
@input=${(e) => this.change(e)}>
|
||||
</div>
|
||||
<div class="filter" id="f-email" style="left: 20%;visibility: ${this.filters.email.visible}">
|
||||
<input id="i-email" placeholder="email" .value=${this.filters.email.value}
|
||||
@input=${(e) => this.change(e)}>
|
||||
</div>
|
||||
<div id="f-streetname" class="filter" style="left: 34%;visibility: ${this.filters.streetname.visible}">
|
||||
<input id="i-streetname" placeholder="streetname" .value=${this.filters.streetname.value}
|
||||
@input=${e => this.change(e)}>
|
||||
</div>
|
||||
<div id="f-housenumber" class="filter"
|
||||
style="left: 50%;visibility: ${this.filters.housenumber.visible}">
|
||||
<input id="i-housenumber" placeholder="housenumber" .value=${this.filters.housenumber.value}
|
||||
@input=${(e) => this.change(e)}>
|
||||
</div>
|
||||
<div id="f-city" class="filter" style="left: 66%;visibility: ${this.filters.city.visible}">
|
||||
<input id="i-city" placeholder="city" .value=${this.filters.city.value}
|
||||
@input=${(e) => this.change(e)}>
|
||||
</div>
|
||||
<div id="f-country" class="filter" style="left: 82%;visibility: ${this.filters.country.visible}">
|
||||
<input id="i-country" placeholder="country" .value=${this.filters.country.value}
|
||||
@input=${(e) => this.change(e)}>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<x-virtual-scroller
|
||||
size="${18}"
|
||||
.items="${this.data}"
|
||||
.idFn="${a => a[0]}"
|
||||
.renderFn="${a => html`
|
||||
<x-item .item="${a}"/>`}">
|
||||
</x-virtual-scroller>
|
||||
<div style="position: absolute;bottom: 10%;left: 10px">${this.data.length} results</div>`;
|
||||
}
|
||||
|
||||
createWhereClause() {
|
||||
return columns
|
||||
.filter(item => this.filters[item].value && this.filters[item].value !== '')
|
||||
.map(item => `${item} like '%${this.filters[item].value}%'`)
|
||||
.join(" AND ");
|
||||
}
|
||||
|
||||
change(e) {
|
||||
this.filters[e.target.id.substring(2)].value = e.target.value;
|
||||
this.dispatchEvent(new CustomEvent('filterupdate-event', {
|
||||
detail: {message: this.createWhereClause()},
|
||||
bubbles: true,
|
||||
composed: true
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
if (!customElements.get('virtual-scroller')) {
|
||||
customElements.define('virtual-scroller', App);
|
||||
}
|
||||
44
demo/ui/src/database.js
Normal file
44
demo/ui/src/database.js
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import sqlite3InitModule from "sqlite-wasm-esm";
|
||||
|
||||
(async () => {
|
||||
let db;
|
||||
|
||||
const error = (...args) => console.log('error', ...args);
|
||||
let sqlite3 = await sqlite3InitModule({printErr: error,});
|
||||
|
||||
const dataBuffer = await fetch("/api/db/customers");
|
||||
try {
|
||||
// save the response in opfs:my.db
|
||||
const root = await navigator.storage.getDirectory();
|
||||
const draftFile = await root.getFileHandle("my.db", {create: true});
|
||||
const accessHandle = await draftFile.createSyncAccessHandle();
|
||||
accessHandle.write(new Uint8Array(await dataBuffer.arrayBuffer()));
|
||||
|
||||
// release the lock, so the database can read the file
|
||||
await accessHandle.close();
|
||||
|
||||
// open database using opfs and my.db
|
||||
db = new sqlite3.opfs.OpfsDb('my.db');
|
||||
postMessage("db ready");
|
||||
} catch (error) {
|
||||
// getting the accessHandle may fail if there are other locks
|
||||
console.log(error);
|
||||
}
|
||||
|
||||
// listens to events from main thread
|
||||
onmessage = (e) => {
|
||||
let data = [];
|
||||
|
||||
db.exec({
|
||||
sql: e.data, // the query
|
||||
callback: function (row) {
|
||||
// fetch the data, row by row
|
||||
// row is [value1, value2, ...]
|
||||
data.push(row);
|
||||
}
|
||||
});
|
||||
|
||||
// pass the data as a message to the main thread
|
||||
postMessage(data);
|
||||
}
|
||||
})();
|
||||
39
demo/ui/src/item.js
Normal file
39
demo/ui/src/item.js
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
import {LitElement, html} from 'lit';
|
||||
|
||||
class Item extends LitElement {
|
||||
render() {
|
||||
return html`
|
||||
<style>
|
||||
:host {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
:host .column {
|
||||
display: inline-block;
|
||||
width: 16%;
|
||||
overflow: hidden
|
||||
}
|
||||
|
||||
.row{
|
||||
border: solid #fff 1px;
|
||||
}
|
||||
</style>
|
||||
<div class="row">
|
||||
<div class="column">${this.item[1]}</div>
|
||||
<div class="column">${this.item[2]}</div>
|
||||
<div class="column">${this.item[3]}</div>
|
||||
<div class="column">${this.item[4]}</div>
|
||||
<div class="column">${this.item[5]}</div>
|
||||
<div class="column">${this.item[6]}</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
Item.properties = () => ({
|
||||
item: {type: Object, attribute: false, reflect: false},
|
||||
});
|
||||
|
||||
if (!customElements.get('x-item')) {
|
||||
customElements.define('x-item', Item);
|
||||
}
|
||||
28
demo/ui/src/result-count.js
Normal file
28
demo/ui/src/result-count.js
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
import {LitElement, html} from "lit";
|
||||
|
||||
class ResultCount extends LitElement {
|
||||
static get properties() {
|
||||
return {
|
||||
count: {type: Number, attribute: false},
|
||||
}
|
||||
}
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.count = 10;
|
||||
this.addEventListener('someEvent', e => {
|
||||
console.log("count");
|
||||
this.count = e.detail.message;
|
||||
});
|
||||
}
|
||||
|
||||
render() {
|
||||
return html`
|
||||
<p>${this.count} results</p>
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
if (!customElements.get('x-result-count')) {
|
||||
customElements.define('x-result-count', ResultCount);
|
||||
}
|
||||
15
demo/ui/src/sort.svg
Normal file
15
demo/ui/src/sort.svg
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
<?xml version="1.0" encoding="iso-8859-1"?>
|
||||
<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<svg fill="#606060" version="1.1" id="Capa_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
|
||||
width="1em" height="1em" viewBox="0 0 36.678 36.678"
|
||||
xml:space="preserve">
|
||||
<g>
|
||||
<path d="M29.694,20.074c0.087,0.16,0.08,0.356-0.02,0.512L19.393,36.449c-0.089,0.139-0.241,0.224-0.406,0.229
|
||||
c-0.004,0-0.009,0-0.014,0c-0.159,0-0.31-0.074-0.403-0.206L6.997,20.609c-0.111-0.152-0.127-0.354-0.042-0.521
|
||||
s0.258-0.273,0.446-0.273h21.855C29.439,19.814,29.608,19.914,29.694,20.074z M7.401,16.864h21.855c0.007,0,0.013,0,0.02,0
|
||||
c0.276,0,0.5-0.224,0.5-0.5c0-0.156-0.069-0.295-0.184-0.387L18.086,0.205C17.989,0.073,17.838,0.009,17.669,0
|
||||
c-0.165,0.005-0.315,0.09-0.406,0.228L6.982,16.092c-0.101,0.154-0.107,0.35-0.021,0.511C7.05,16.764,7.218,16.864,7.401,16.864z"
|
||||
/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1 KiB |
38
demo/ui/src/styles.css
Normal file
38
demo/ui/src/styles.css
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
body {
|
||||
font-family: sans-serif;
|
||||
}
|
||||
|
||||
virtual-scroller {
|
||||
display: flex;
|
||||
height: 80vh;
|
||||
width: 100%;
|
||||
overflow: scroll;
|
||||
}
|
||||
|
||||
.header {
|
||||
height: 1em;
|
||||
display: inline-block;
|
||||
width: 15.9%;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
img {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.filter {
|
||||
cursor: pointer;
|
||||
float: right;
|
||||
color: gray;
|
||||
font-size: larger;
|
||||
transform: translate(0, -3px);
|
||||
}
|
||||
|
||||
img {
|
||||
width: 1em;
|
||||
height: 0.7em;
|
||||
}
|
||||
|
||||
.header-row {
|
||||
border: solid #eee 1px;
|
||||
}
|
||||
56
demo/ui/src/user-filter.js
Normal file
56
demo/ui/src/user-filter.js
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import {LitElement, html} from "lit";
|
||||
|
||||
class UserFilter extends LitElement {
|
||||
static get properties() {
|
||||
return {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.name = "";
|
||||
this.email = "";
|
||||
this.streetname = "";
|
||||
this.housenumber = "";
|
||||
this.city = "";
|
||||
this.country = "";
|
||||
}
|
||||
|
||||
createWhereClause() {
|
||||
return ['name', 'email', 'streetname', 'housenumber', 'city', 'country']
|
||||
.filter(item => this[item] && this[item] !== '')
|
||||
.map(item => `${item} like '%${this[item]}%'`)
|
||||
.join(" AND ");
|
||||
}
|
||||
|
||||
change(e) {
|
||||
this[e.target.id] = e.target.value;
|
||||
this.dispatchEvent(new CustomEvent('filterupdate-event', {
|
||||
detail: {message: this.createWhereClause()},
|
||||
bubbles: true,
|
||||
composed: true
|
||||
}));
|
||||
}
|
||||
|
||||
render() {
|
||||
return html`
|
||||
<style>
|
||||
:host {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.header {
|
||||
height: 1em;
|
||||
display: inline-block;
|
||||
width: 16%;
|
||||
}
|
||||
</style>
|
||||
`;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (!customElements.get('x-user-filter')) {
|
||||
customElements.define('x-user-filter', UserFilter);
|
||||
}
|
||||
86
demo/ui/src/virtual-scroll.js
Normal file
86
demo/ui/src/virtual-scroll.js
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import {LitElement, html} from 'lit';
|
||||
import {repeat} from 'lit/directives/repeat.js';
|
||||
import './user-filter.js';
|
||||
|
||||
class VirtualScroller extends LitElement {
|
||||
constructor() {
|
||||
super();
|
||||
this.currentBase = this.currentBase === undefined ? 0 : this.currentBase;
|
||||
this.onScroll = this.onScroll.bind(this);
|
||||
this.size = this.size || 18;
|
||||
}
|
||||
|
||||
connectedCallback() {
|
||||
super.connectedCallback();
|
||||
this.addEventListener('scroll', this.onScroll);
|
||||
this.wrapperHeight = this.getClientRects()[0].height;
|
||||
}
|
||||
|
||||
onScroll(event) {
|
||||
if (this.scrollTicking) {
|
||||
window.cancelAnimationFrame(this.scrollTicking);
|
||||
this.scrollTicking = null;
|
||||
}
|
||||
this.scrollTicking = window.requestAnimationFrame(() => {
|
||||
this.currentBase = Math.round(this.scrollTop / this.size);
|
||||
});
|
||||
}
|
||||
|
||||
register(inputElements){
|
||||
inputElements.addEventListener("change");
|
||||
}
|
||||
|
||||
render() {
|
||||
const listSize = Math.round(this.wrapperHeight / this.size) + 2;
|
||||
const filteredList = this.items.slice(
|
||||
this.currentBase,
|
||||
this.currentBase + listSize
|
||||
);
|
||||
this.style.setProperty(
|
||||
'--vs-top-height',
|
||||
`${this.currentBase * this.size}px`
|
||||
);
|
||||
this.style.setProperty(
|
||||
'--vs-bottom-heig ht',
|
||||
`${(this.items.length - listSize - this.currentBase) * this.size}px`
|
||||
);
|
||||
return html`
|
||||
<style>
|
||||
:host {
|
||||
border: solid #eee 1px;
|
||||
margin-top: 5px;
|
||||
padding: 3px;
|
||||
position: relative;
|
||||
overflow: scroll;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
:host .top-padding {
|
||||
height: var(--vs-top-height);
|
||||
}
|
||||
|
||||
:host .bottom-padding {
|
||||
height: var(--vs-bottom-height);
|
||||
}
|
||||
</style>
|
||||
<div class="top-padding"></div>
|
||||
|
||||
${repeat(filteredList, this.idFn, this.renderFn)}
|
||||
<div class="bottom-padding"></div>
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
Object.defineProperty(VirtualScroller, 'properties', {
|
||||
get: () => ({
|
||||
size: {type: Number, attribute: true, reflect: true},
|
||||
currentBase: {type: Number, attribute: true, reflect: true},
|
||||
items: {type: Array, attribute: false, reflect: false},
|
||||
idFn: {type: Function, attribute: false, reflect: false},
|
||||
renderFn: {type: Function, attribute: false, reflect: false},
|
||||
}),
|
||||
});
|
||||
|
||||
if (!customElements.get('x-virtual-scroller')) {
|
||||
customElements.define('x-virtual-scroller', VirtualScroller);
|
||||
}
|
||||
27
demo/ui/vite.config.js
Normal file
27
demo/ui/vite.config.js
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import { defineConfig } from 'vite';
|
||||
import fs from 'fs';
|
||||
|
||||
export default defineConfig({
|
||||
optimizeDeps: {
|
||||
exclude: ['sqlite-wasm-esm'], // TODO remove once fixed https://github.com/vitejs/vite/issues/8427
|
||||
},
|
||||
dbWorker: {
|
||||
format: 'es'
|
||||
},
|
||||
server: {
|
||||
https: {
|
||||
key: fs.readFileSync('./.cert/key.pem'),
|
||||
cert: fs.readFileSync('./.cert/cert.pem'),
|
||||
},
|
||||
headers: {
|
||||
"Cross-Origin-Opener-Policy": "same-origin",
|
||||
"Cross-Origin-Embedder-Policy": "require-corp"
|
||||
},
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://localhost:8080',
|
||||
changeOrigin: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
46
fileviewer/pom.xml
Normal file
46
fileviewer/pom.xml
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>sqlighter-pom</artifactId>
|
||||
<groupId>nl.sanderhautvast</groupId>
|
||||
<version>1.1.1</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>fileviewer</artifactId>
|
||||
<version>1.1.1</version>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>nl.sanderhautvast</groupId>
|
||||
<artifactId>sqlighter</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>2.14.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.projectlombok</groupId>
|
||||
<artifactId>lombok</artifactId>
|
||||
<version>1.18.26</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter-engine</artifactId>
|
||||
<version>5.9.2</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.Metadatabase;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.page.PageReader;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.page.RootpageReader;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* Outputs the lowlevel structure of the database file as json object.
|
||||
* Meant to be piped into jq / other.
|
||||
* <p>
|
||||
* <p>
|
||||
* Can/should be extended to find more corruption in sqlite files.
|
||||
*/
|
||||
public class Main {
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
String filename = args[0]; //TODO cmdline arg validation
|
||||
|
||||
// first 100 bytes contain the header part with the actual pagesize used, which is what we'll use as
|
||||
// size of the bytebuffer as well. So first we read 100 bytes, then reopen the file
|
||||
// reading the whole page (excluding the header) and then the following pages
|
||||
Metadatabase metadatabase = readRootPage(filename);
|
||||
|
||||
ByteBuffer buffer = ByteBuffer.allocate(metadatabase.getPagesize());
|
||||
PageReader pageReader = new PageReader(metadatabase);
|
||||
|
||||
try (FileInputStream file = new FileInputStream(filename)) {
|
||||
for (long pageNumber = 1; ; pageNumber += 1) {
|
||||
buffer.position(0);
|
||||
int nread = file.getChannel().read(buffer);
|
||||
|
||||
if (nread == 0) {
|
||||
break; //regular end
|
||||
} else if (nread < metadatabase.getPagesize()) {
|
||||
break;
|
||||
} else {
|
||||
buffer.position(0);
|
||||
if (pageNumber == 1) {
|
||||
SchemaReader schemaReader = new SchemaReader(metadatabase);
|
||||
metadatabase.getSchemaRecords().addAll(schemaReader.readSchema(buffer));
|
||||
} else {
|
||||
pageReader.readPage(buffer, pageNumber);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
System.out.println(metadatabase.report());
|
||||
}
|
||||
|
||||
private static Metadatabase readRootPage(String filename) throws IOException {
|
||||
try (FileInputStream file = new FileInputStream(filename)) {
|
||||
ByteBuffer headerDataBUffer = ByteBuffer.allocate(0x100);
|
||||
file.getChannel().read(headerDataBUffer);
|
||||
return RootpageReader.read(headerDataBUffer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.Varint;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.data.*;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.data.ReadOnlyRecord;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.Metadatabase;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class RecordReader {
|
||||
|
||||
private final Metadatabase metadatabase;
|
||||
|
||||
public RecordReader(Metadatabase metadatabase) {
|
||||
this.metadatabase = metadatabase;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused") // leaving bits in, without using them, because it works as documentation
|
||||
public ReadOnlyRecord read(ByteBuffer buffer) {
|
||||
long payloadLength = Varint.read(buffer);
|
||||
long rowId = Varint.read(buffer);
|
||||
long columnLengthSum = Varint.read(buffer);
|
||||
|
||||
int mark = buffer.position();
|
||||
|
||||
List<Long> columnTypes = new ArrayList<>();
|
||||
while (buffer.position() < mark + columnLengthSum -1) {
|
||||
columnTypes.add(Varint.read(buffer));
|
||||
}
|
||||
|
||||
ReadOnlyRecord record = new ReadOnlyRecord(rowId);
|
||||
for (long columnType : columnTypes) {
|
||||
record.addValue(ReadOnlyValue.read(buffer, columnType, metadatabase.getEncoding().toCharset()));
|
||||
}
|
||||
|
||||
return record;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.SchemaRecord;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.data.ReadOnlyRecord;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.Metadatabase;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.Table;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static nl.sanderhautvast.sqlighter.fileviewer.util.UnsignedIntReader.readU16;
|
||||
import static nl.sanderhautvast.sqlighter.fileviewer.util.UnsignedIntReader.readU8;
|
||||
|
||||
public class SchemaReader {
|
||||
private final Metadatabase meta;
|
||||
|
||||
public SchemaReader(Metadatabase meta) {
|
||||
this.meta = meta;
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
// leaving bits in, without using them, because it may work as documentation
|
||||
List<SchemaRecord> readSchema(ByteBuffer buffer) {
|
||||
// skip page type 0x0D at #100
|
||||
buffer.position(101);
|
||||
buffer.limit(meta.getPagesize());
|
||||
|
||||
long freeBlockStart = readU16(buffer);
|
||||
long nrCells = readU16(buffer);
|
||||
int startOfContentArea = readU16(buffer);
|
||||
long nrFragmentedFreeBytes = readU8(buffer);
|
||||
|
||||
buffer.position(startOfContentArea);
|
||||
List<SchemaRecord> records = new ArrayList<>();
|
||||
RecordReader recordReader = new RecordReader(meta);
|
||||
|
||||
for (int i = 0; i < nrCells; i++) {
|
||||
ReadOnlyRecord record = recordReader.read(buffer);
|
||||
String name = record.getStringValue(1).getValue();
|
||||
long rootPage = record.getIntegerValue(3).getValue().intValue();
|
||||
String sqlObjectType = record.getStringValue(0).getValue();
|
||||
|
||||
String sql = record.getStringValue(4).getValue();
|
||||
records.add(new SchemaRecord(record.getRowId(), name, rootPage, sql));
|
||||
meta.addTable(new Table(name, rootPage));
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.data;
|
||||
|
||||
|
||||
import nl.sanderhautvast.sqlighter.Varint;
|
||||
|
||||
import java.nio.BufferUnderflowException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
public class BlobValue extends ReadOnlyValue<byte[]> {
|
||||
|
||||
public BlobValue(byte[] value) {
|
||||
super(Varint.write(value.length * 2L + 12), value);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public byte[] getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
@Override
|
||||
int getLength() {
|
||||
return type.length + value.length;
|
||||
}
|
||||
|
||||
public static BlobValue read(int length, ByteBuffer in) {
|
||||
byte[] bytes = new byte[length];
|
||||
try {
|
||||
in.get(bytes);
|
||||
return new BlobValue(bytes);
|
||||
} catch (BufferUnderflowException e) {
|
||||
throw new IllegalStateException("should have read " + length + " bytes", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.data;
|
||||
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* 64-bit float type
|
||||
*/
|
||||
public class FloatValue extends ReadOnlyValue<Double> {
|
||||
|
||||
final private double externalValue;
|
||||
private static final byte FLOAT_TYPE = 7;
|
||||
|
||||
|
||||
public FloatValue(double value) {
|
||||
super(new byte[]{FLOAT_TYPE}, ByteBuffer.wrap(new byte[8]).putDouble(0, value).array());
|
||||
this.externalValue = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Double getValue() {
|
||||
return externalValue;
|
||||
}
|
||||
|
||||
@Override
|
||||
int getLength() {
|
||||
return 9; // 1 for type, 8 for 64 bits
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.data;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.data.LtValue;
|
||||
|
||||
/*
|
||||
* Uses long (s64) as standard integer representation
|
||||
*/
|
||||
public class IntegerValue extends ReadOnlyValue<Long> {
|
||||
|
||||
private final long externalValue;
|
||||
|
||||
public IntegerValue(long value) {
|
||||
super(ReadOnlyValue.getIntegerType(value), LtValue.getValueAsBytes(value));
|
||||
this.externalValue = value; //See StringValue
|
||||
}
|
||||
|
||||
public IntegerValue(byte[] typeByteRep, byte[] valueByteRep) {
|
||||
super(typeByteRep, valueByteRep);
|
||||
this.externalValue = bytesToLong(valueByteRep);
|
||||
}
|
||||
|
||||
@Override
|
||||
int getLength() {
|
||||
return type.length + value.length;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public Long getValue() {
|
||||
return externalValue;
|
||||
}
|
||||
|
||||
public static long bytesToLong(final byte[] b) {
|
||||
long n = 0;
|
||||
for (int i = 0; i < b.length; i++) {
|
||||
byte v = b[i];
|
||||
int shift = ((b.length - i - 1) * 8);
|
||||
if (i == 0 && (v & 0x80) != 0) {
|
||||
n -= (0x80L << shift);
|
||||
v &= 0x7f;
|
||||
}
|
||||
n += ((long)(v&0xFF)) << shift;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Record in sqlite database.
|
||||
* Used for reading and writing.
|
||||
*/
|
||||
public class ReadOnlyRecord {
|
||||
|
||||
|
||||
private final long rowId;
|
||||
|
||||
private final List<ReadOnlyValue<?>> values = new ArrayList<>();
|
||||
|
||||
public ReadOnlyRecord(long rowId) {
|
||||
this.rowId = rowId;
|
||||
}
|
||||
|
||||
public void addValue(ReadOnlyValue<?> value) {
|
||||
this.values.add(value);
|
||||
}
|
||||
|
||||
public int length() {
|
||||
return values.stream().mapToInt(ReadOnlyValue::getLength).sum() + 1;
|
||||
}
|
||||
|
||||
public long getRowId() {
|
||||
return rowId;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
public List<ReadOnlyValue<?>> getValues() {
|
||||
return values;
|
||||
}
|
||||
|
||||
/**
|
||||
* returns the value at the specified column index (0 based)
|
||||
*/
|
||||
public ReadOnlyValue<?> getValue(int column) {
|
||||
return values.get(column);
|
||||
}
|
||||
|
||||
public StringValue getStringValue(int column) {
|
||||
ReadOnlyValue<?> value = getValue(column);
|
||||
if (value instanceof StringValue) {
|
||||
return (StringValue) value;
|
||||
} else {
|
||||
throw new IllegalCallerException("value is not a StringValue, but a " + value.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
|
||||
public IntegerValue getIntegerValue(int column) {
|
||||
ReadOnlyValue<?> value = getValue(column);
|
||||
if (value instanceof IntegerValue) {
|
||||
return (IntegerValue) value;
|
||||
} else {
|
||||
throw new IllegalCallerException("value is not a IntegerValue, but a " + value.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
ReadOnlyRecord record = (ReadOnlyRecord) o;
|
||||
return rowId == record.rowId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(rowId);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.data;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.Varint;
|
||||
import nl.sanderhautvast.sqlighter.data.LtValue;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.Charset;
|
||||
|
||||
/*
|
||||
* NB Value classes derive their equality from their identity. I.e. no equals/hashcode
|
||||
*/
|
||||
public abstract class ReadOnlyValue<T> {
|
||||
|
||||
protected final byte[] type;
|
||||
protected final byte[] value;
|
||||
|
||||
protected ReadOnlyValue(byte[] type, byte[] value) {
|
||||
this.type = type;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* returns the user representation of the value
|
||||
*/
|
||||
public abstract T getValue();
|
||||
|
||||
/**
|
||||
* Returns the length of serialType + the length of the value
|
||||
*/
|
||||
abstract int getLength();
|
||||
|
||||
/**
|
||||
* Reads a value from the buffer
|
||||
*
|
||||
* @param buffer Bytebuffer containing the database page.
|
||||
* @param columnType sqlite type representation
|
||||
* @param charset database charset
|
||||
*
|
||||
* @return the value implementation
|
||||
*/
|
||||
public static ReadOnlyValue<?> read(ByteBuffer buffer, long columnType, Charset charset) {
|
||||
if (columnType == 0) {
|
||||
return null;
|
||||
} else if (columnType < 6L) {
|
||||
byte[] integerBytes = new byte[getvalueLengthForType(columnType)];
|
||||
buffer.get(integerBytes);
|
||||
//TODO columnType is first decoded, then encoded again
|
||||
return new IntegerValue(Varint.write(columnType), integerBytes);
|
||||
} else if (columnType == 7) {
|
||||
return new FloatValue(buffer.getDouble());
|
||||
} else if (columnType == 8) {
|
||||
return new IntegerValue(0);
|
||||
} else if (columnType == 9) {
|
||||
return new IntegerValue(1);
|
||||
} else if (columnType >= 12 && columnType % 2 == 0) {
|
||||
return BlobValue.read(getvalueLengthForType(columnType), buffer);
|
||||
} else if (columnType >= 13) {
|
||||
return StringValue.read(getvalueLengthForType(columnType), buffer, charset);
|
||||
} else throw new IllegalStateException("unknown column type" + columnType);
|
||||
}
|
||||
|
||||
private static int getvalueLengthForType(long columnType) {
|
||||
// can't switch on long
|
||||
if (columnType == 0 || columnType == 8 || columnType == 9) {
|
||||
return 0;
|
||||
} else if (columnType < 5) {
|
||||
return (int) columnType;
|
||||
} else if (columnType == 5) {
|
||||
return 6;
|
||||
} else if (columnType == 6 || columnType == 7) {
|
||||
return 8;
|
||||
} else if (columnType < 12) {
|
||||
return -1;
|
||||
} else {
|
||||
if (columnType % 2 == 0) {
|
||||
return (int) ((columnType - 12) >> 1);
|
||||
} else {
|
||||
return (int) ((columnType - 13) >> 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static byte[] getIntegerType(long value) {
|
||||
if (value == 0) {
|
||||
return new byte[]{8};
|
||||
} else if (value == 1) {
|
||||
return new byte[]{9};
|
||||
} else {
|
||||
int length = LtValue.getLengthOfByteEncoding(value);
|
||||
if (length < 5) {
|
||||
return Varint.write(length);
|
||||
} else if (length < 7) {
|
||||
return Varint.write(5);
|
||||
} else return Varint.write(6);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.data;
|
||||
|
||||
|
||||
import nl.sanderhautvast.sqlighter.Varint;
|
||||
|
||||
import java.nio.BufferUnderflowException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
public class StringValue extends ReadOnlyValue<String> {
|
||||
private final String externalValue;
|
||||
|
||||
public StringValue(String value) {
|
||||
super(Varint.write(value == null ? 0 : value.getBytes(StandardCharsets.UTF_8).length * 2L + 13),
|
||||
value == null ? new byte[0] : value.getBytes(StandardCharsets.UTF_8));
|
||||
this.externalValue = value; /* only for reading from db. could be optimized by not storing this when writing,
|
||||
or create separate value classes for reading and writing */
|
||||
}
|
||||
|
||||
@Override
|
||||
int getLength() {
|
||||
return type.length + value.length;
|
||||
}
|
||||
|
||||
public static StringValue read(int length, ByteBuffer in, Charset charset) {
|
||||
byte[] bytes = new byte[length];
|
||||
try {
|
||||
in.get(bytes);
|
||||
return new StringValue(new String(bytes, charset));
|
||||
} catch (BufferUnderflowException e) {
|
||||
throw new IllegalStateException("should have read " + length + " bytes", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getValue() {
|
||||
return externalValue;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.model;
|
||||
|
||||
public class Cell {
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.model;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
public class Index {
|
||||
private final String name;
|
||||
private final List<IndexCell> indexCells = new ArrayList<>();
|
||||
|
||||
public void addCell(IndexCell indexCell) {
|
||||
indexCells.add(indexCell);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.model;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.data.ReadOnlyValue;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public class IndexCell extends Cell{
|
||||
private int valueType;
|
||||
private int rowIdType;
|
||||
private final List<ReadOnlyValue<?>> indexValues = new ArrayList<>();
|
||||
|
||||
/* reference to the row in the table */
|
||||
private long rowid;
|
||||
|
||||
public void addIndexValue(ReadOnlyValue<?> value) {
|
||||
this.indexValues.add(value);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public class InteriorCell extends Cell{
|
||||
@JsonIgnore
|
||||
private long pageReference;
|
||||
@JsonIgnore
|
||||
private long lastRowId;
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.model;
|
||||
|
||||
public class LeafCell extends Cell{
|
||||
}
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.Data;
|
||||
import nl.sanderhautvast.sqlighter.SchemaRecord;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.util.Encoding;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.util.Printer;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
import static nl.sanderhautvast.sqlighter.fileviewer.validation.Validator.validate;
|
||||
|
||||
@Data
|
||||
public class Metadatabase {
|
||||
|
||||
public static final String HEADER = "53 51 4C 69 74 65 20 66 6F 72 6D 61 74 20 33 00";
|
||||
public static final int MAX_EMBEDDED_PAYLOAD_FRACTION = 64;
|
||||
public static final int MIN_EMBEDDED_PAYLOAD_FRACTION = 32;
|
||||
public static final int LEAF_PAYLOAD_FRACTION = 32;
|
||||
|
||||
private String magicHeader;
|
||||
private int pagesize;
|
||||
private int writeversion;
|
||||
private int readversion;
|
||||
private int unusedPerPageSize;
|
||||
private int maxEmbeddedPayloadFraction;
|
||||
private int minEmbeddedPayloadFraction;
|
||||
private int leafPayloadFraction;
|
||||
private long fileChangeCounter;
|
||||
private long sizeInPages;
|
||||
private long pageNrFirstFreelistTrunkpage;
|
||||
private long totalNrOfFreelistPages;
|
||||
private long schemaCookie;
|
||||
private long schemaFormatNumber;
|
||||
private int defaultPageCachesize;
|
||||
private long pageNrLargestRootBtreePage;
|
||||
private Encoding encoding;
|
||||
private long userVersion;
|
||||
private boolean incrementalVacuumMode;
|
||||
private long applicationID;
|
||||
private String expansion;
|
||||
private long versionValidFor;
|
||||
private long sqliteVersion;
|
||||
|
||||
private final List<SchemaRecord> schemaRecords = new ArrayList<>();
|
||||
|
||||
@JsonIgnore
|
||||
private HashMap<Long, PageNode> pages = new HashMap<>();
|
||||
|
||||
public void addPage(long nr, PageNode page) {
|
||||
pages.put(nr, page);
|
||||
}
|
||||
|
||||
// @JsonIgnore
|
||||
private final List<Table> tables = new ArrayList<>();
|
||||
|
||||
public void addTable(Table table) {
|
||||
tables.add(table);
|
||||
}
|
||||
|
||||
public void setMagicHeader(byte[] magicHeader) {
|
||||
this.magicHeader = Printer.printHex(magicHeader);
|
||||
validate("magic header", HEADER, this.magicHeader);
|
||||
}
|
||||
|
||||
public void setMaxEmbeddedPayloadFraction(int maxEmbeddedPayloadFraction) {
|
||||
this.maxEmbeddedPayloadFraction = maxEmbeddedPayloadFraction;
|
||||
validate("max embedded payload fraction", MAX_EMBEDDED_PAYLOAD_FRACTION, maxEmbeddedPayloadFraction);
|
||||
}
|
||||
|
||||
public void setMinEmbeddedPayloadFraction(int minEmbeddedPayloadFraction) {
|
||||
validate("min embedded payload fraction", MIN_EMBEDDED_PAYLOAD_FRACTION, minEmbeddedPayloadFraction);
|
||||
this.minEmbeddedPayloadFraction = minEmbeddedPayloadFraction;
|
||||
}
|
||||
|
||||
public void setLeafPayloadFraction(int leafPayloadFraction) {
|
||||
this.leafPayloadFraction = leafPayloadFraction;
|
||||
validate("leaf payload fraction", LEAF_PAYLOAD_FRACTION, leafPayloadFraction);
|
||||
}
|
||||
|
||||
public String report() throws IOException {
|
||||
return new ObjectMapper().writeValueAsString(this);
|
||||
}
|
||||
|
||||
public void setExpansion(byte[] expansion) {
|
||||
this.expansion = Printer.printHex(expansion);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import lombok.Data;
|
||||
import nl.sanderhautvast.sqlighter.page.PageType;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@JsonInclude(JsonInclude.Include.NON_EMPTY)
|
||||
public class PageNode {
|
||||
private final long number;
|
||||
private PageType type;
|
||||
private long cellCount;
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
private final List<Cell> cells = new ArrayList<>();
|
||||
|
||||
private final List<PageNode> childPages = new ArrayList<>();
|
||||
|
||||
@JsonIgnore
|
||||
private Long rightMostReference;
|
||||
|
||||
private PageNode rightMostChildPage;
|
||||
|
||||
public PageNode(long number) {
|
||||
this.number = number;
|
||||
}
|
||||
|
||||
public void addCell(Cell cell) {
|
||||
cells.add(cell);
|
||||
}
|
||||
|
||||
public void addChildPage(PageNode page) {
|
||||
childPages.add(page);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import lombok.Data;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.data.ReadOnlyRecord;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
public class Table {
|
||||
private final String name;
|
||||
private final long rootPage;
|
||||
@JsonIgnore
|
||||
private final List<ReadOnlyRecord> records = new ArrayList<>();
|
||||
private int nRecords = 0;
|
||||
|
||||
@JsonIgnore
|
||||
private final List<InteriorCell> interiorCells = new ArrayList<>();
|
||||
|
||||
public void addRecord(ReadOnlyRecord record) {
|
||||
records.add(record);
|
||||
nRecords += 1;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.page;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.Metadatabase;
|
||||
|
||||
//TODO implement
|
||||
public class IndexInteriorPageReader {
|
||||
private final Metadatabase metadatabase;
|
||||
|
||||
public IndexInteriorPageReader(Metadatabase metadatabase) {
|
||||
this.metadatabase = metadatabase;
|
||||
}
|
||||
|
||||
public void readPage() {
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.page;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.Varint;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.data.ReadOnlyValue;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.*;
|
||||
import nl.sanderhautvast.sqlighter.page.PageType;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.util.UnsignedIntReader;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.ArrayList;
|
||||
|
||||
/*
|
||||
* WIP
|
||||
*/
|
||||
public class IndexLeafPageReader {
|
||||
private final Metadatabase metadatabase;
|
||||
private final Charset charset; //stored so that it doesn't have to be retrieved every time
|
||||
|
||||
public IndexLeafPageReader(Metadatabase metadatabase) {
|
||||
this.metadatabase = metadatabase;
|
||||
this.charset = metadatabase.getEncoding().toCharset();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused") // leaving bits in, without using them, because it works as documentation
|
||||
public void readPage(ByteBuffer buffer, long pageNumber) {
|
||||
int freeblockStartLocation = UnsignedIntReader.readU16(buffer);
|
||||
int nrCells = UnsignedIntReader.readU16(buffer);
|
||||
int startOfContentArea = UnsignedIntReader.readU16(buffer);
|
||||
int nrFragmentedFreeBytes = UnsignedIntReader.readU8(buffer);
|
||||
|
||||
PageNode newPage=new PageNode(pageNumber);
|
||||
newPage.setType(PageType.INDEX_LEAF);
|
||||
newPage.setCellCount(nrCells);
|
||||
|
||||
for (int i = 0; i < nrCells; i++) {
|
||||
int cellOffset = UnsignedIntReader.readU16(buffer);
|
||||
int mark = buffer.position();
|
||||
|
||||
buffer.position(cellOffset);
|
||||
newPage.addCell(read(buffer));
|
||||
buffer.position(mark);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* has a similar structure as RecordReader.read, but the rowId needs to be handled differently
|
||||
*/
|
||||
@SuppressWarnings("unused") // leaving bits in, without using them, because it works as documentation
|
||||
public IndexCell read(ByteBuffer buffer) {
|
||||
IndexCell indexCell = new IndexCell();
|
||||
long payloadLength = Varint.read(buffer);
|
||||
long startOfValues = Varint.read(buffer);
|
||||
int mark = buffer.position();
|
||||
|
||||
ArrayList<Integer> columnTypes = new ArrayList<>();
|
||||
while (buffer.position() < mark + startOfValues -1) {
|
||||
columnTypes.add((int) Varint.read(buffer));
|
||||
}
|
||||
|
||||
for (int columnType : columnTypes) {
|
||||
indexCell.addIndexValue(ReadOnlyValue.read(buffer, columnType, charset));
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked", "ConstantConditions"})
|
||||
Long rowid = ((ReadOnlyValue<Long>) ReadOnlyValue.read(buffer, columnTypes.get(columnTypes.size() - 1), charset)).getValue();
|
||||
indexCell.setRowid(rowid);
|
||||
|
||||
return indexCell;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.page;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.SQLiteConstants;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.Metadatabase;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.util.UnsignedIntReader;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
public class PageReader {
|
||||
|
||||
private final Metadatabase metadatabase;
|
||||
|
||||
public PageReader(Metadatabase metadatabase) {
|
||||
this.metadatabase = metadatabase;
|
||||
}
|
||||
|
||||
public void readPage(ByteBuffer buffer, long pageNumber) {
|
||||
int pagetype = UnsignedIntReader.readU8(buffer);
|
||||
switch (pagetype) {
|
||||
case SQLiteConstants.TABLE_LEAF_PAGE: new TableLeafPageReader(metadatabase).readPage(buffer, pageNumber);
|
||||
break;
|
||||
case SQLiteConstants.TABLE_INTERIOR_PAGE: new TableInteriorPageReader(metadatabase).readPage(buffer, pageNumber);
|
||||
break;
|
||||
case SQLiteConstants.INDEX_LEAF_PAGE: new IndexLeafPageReader(metadatabase).readPage(buffer, pageNumber);
|
||||
break;
|
||||
case SQLiteConstants.INDEX_INTERIOR_PAGE: new IndexInteriorPageReader(metadatabase).readPage();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.page;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.Metadatabase;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.util.Encoding;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import static nl.sanderhautvast.sqlighter.fileviewer.util.UnsignedIntReader.*;
|
||||
|
||||
public class RootpageReader {
|
||||
|
||||
public static Metadatabase read(ByteBuffer in) {
|
||||
in.position(0); //seems clearer to me than using reset/flip etc
|
||||
in.limit(100);
|
||||
|
||||
Metadatabase m = new Metadatabase();
|
||||
m.setMagicHeader(readBytes(in, 16));
|
||||
m.setPagesize(readU16(in));
|
||||
m.setWriteversion(readU8(in));
|
||||
m.setReadversion(readU8(in));
|
||||
m.setUnusedPerPageSize(readU8(in));
|
||||
m.setMaxEmbeddedPayloadFraction(readU8(in));
|
||||
m.setMinEmbeddedPayloadFraction(readU8(in));
|
||||
m.setLeafPayloadFraction(readU8(in));
|
||||
m.setFileChangeCounter(readU32(in));
|
||||
m.setSizeInPages(readU32(in));
|
||||
m.setPageNrFirstFreelistTrunkpage(readU32(in));
|
||||
m.setTotalNrOfFreelistPages(readU32(in));
|
||||
m.setSchemaCookie(readU32(in));
|
||||
m.setSchemaCookie(readU32(in));
|
||||
m.setDefaultPageCachesize(in.getInt()); // it's signed say the docs
|
||||
m.setPageNrLargestRootBtreePage(readU32(in));
|
||||
m.setEncoding(Encoding.fromCode((int) readU32(in))); // no truncate expected
|
||||
m.setUserVersion(readU32(in));
|
||||
m.setIncrementalVacuumMode(readU32(in) > 0);
|
||||
m.setApplicationID(readU32(in));
|
||||
m.setExpansion(readBytes(in, 20));
|
||||
m.setVersionValidFor(readU32(in));
|
||||
m.setSqliteVersion(readU32(in));
|
||||
return m;
|
||||
}
|
||||
|
||||
static byte[] readBytes(ByteBuffer in, int length) {
|
||||
byte[] header = new byte[length];
|
||||
in.get(header);
|
||||
return header;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.page;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.Varint;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.util.UnsignedIntReader;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.InteriorCell;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.Metadatabase;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.PageNode;
|
||||
import nl.sanderhautvast.sqlighter.page.PageType;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
public class TableInteriorPageReader {
|
||||
private final Metadatabase metadatabase;
|
||||
|
||||
public TableInteriorPageReader(Metadatabase metadatabase) {
|
||||
this.metadatabase = metadatabase;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused") // leaving bits in, without using them, because it works as documentation
|
||||
public void readPage(ByteBuffer buffer, long pageNumber) {
|
||||
int freeblockStartLocation = UnsignedIntReader.readU16(buffer);
|
||||
int nrCells = UnsignedIntReader.readU16(buffer);
|
||||
int startOfContentArea = UnsignedIntReader.readU16(buffer);
|
||||
int nrFragmentedFreeBytes = UnsignedIntReader.readU8(buffer);
|
||||
long rightMostPointer = UnsignedIntReader.readU32(buffer); //?
|
||||
|
||||
|
||||
try {
|
||||
PageNode newPage = new PageNode(pageNumber);
|
||||
newPage.setType(PageType.TABLE_INTERIOR);
|
||||
newPage.setCellCount(nrCells);
|
||||
newPage.setRightMostReference(rightMostPointer);
|
||||
metadatabase.addPage(pageNumber,newPage);
|
||||
|
||||
|
||||
for (int i = 0; i < nrCells; i++) {
|
||||
int cellOffset = UnsignedIntReader.readU16(buffer);
|
||||
int mark = buffer.position();
|
||||
|
||||
InteriorCell interiorCell = new InteriorCell();
|
||||
buffer.position(cellOffset);
|
||||
long childPageNumber = UnsignedIntReader.readU32(buffer);
|
||||
interiorCell.setPageReference(childPageNumber);
|
||||
interiorCell.setLastRowId(Varint.read(buffer));
|
||||
|
||||
newPage.addCell(interiorCell);
|
||||
buffer.position(mark);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.err.printf("Error at page number %d index %d (%#06x): ", pageNumber, buffer.position(), buffer.position());
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.page;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.Metadatabase;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.RecordReader;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.PageNode;
|
||||
import nl.sanderhautvast.sqlighter.page.PageType;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.model.Table;
|
||||
import nl.sanderhautvast.sqlighter.fileviewer.util.UnsignedIntReader;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
public class TableLeafPageReader {
|
||||
private final Metadatabase metadatabase;
|
||||
|
||||
public TableLeafPageReader(Metadatabase metadatabase) {
|
||||
this.metadatabase = metadatabase;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused") // leaving bits in, without using them, because it works as documentation
|
||||
public void readPage(ByteBuffer buffer, long pageNumber) {
|
||||
int freeblockStartLocation = UnsignedIntReader.readU16(buffer);
|
||||
int nrCells = UnsignedIntReader.readU16(buffer);
|
||||
int startOfContentArea = UnsignedIntReader.readU16(buffer);
|
||||
int nrFragmentedFreeBytes = UnsignedIntReader.readU8(buffer);
|
||||
RecordReader recordReader = new RecordReader(metadatabase);
|
||||
|
||||
PageNode newPage = new PageNode(pageNumber);
|
||||
newPage.setType(PageType.TABLE_LEAF);
|
||||
newPage.setCellCount(nrCells);
|
||||
metadatabase.addPage(pageNumber, newPage);
|
||||
}
|
||||
|
||||
|
||||
// reads the actual records, want we that?
|
||||
// make cmdline arg
|
||||
private static void readPage(ByteBuffer buffer, RecordReader recordReader, Table table) {
|
||||
int cellOffset = UnsignedIntReader.readU16(buffer);
|
||||
int mark = buffer.position();
|
||||
|
||||
buffer.position(cellOffset);
|
||||
table.addRecord(recordReader.read(buffer));
|
||||
buffer.position(mark);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.util;
|
||||
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
public enum Encoding implements HasCode {
|
||||
|
||||
UTF_8(1, StandardCharsets.UTF_8),
|
||||
UTF_16LE(2, StandardCharsets.UTF_16LE),
|
||||
UTF_16BE(3, StandardCharsets.UTF_16BE);
|
||||
|
||||
|
||||
private final int code;
|
||||
private final Charset charset;
|
||||
|
||||
Encoding(int code, Charset charset) {
|
||||
this.code = code;
|
||||
this.charset = charset;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getCode() {
|
||||
return code;
|
||||
}
|
||||
|
||||
public static Encoding fromCode(int code) {
|
||||
return HasCode.getFromCode(Encoding.class, code);
|
||||
}
|
||||
|
||||
public Charset toCharset() {
|
||||
return charset;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.util;
|
||||
|
||||
public interface HasCode {
|
||||
static <E extends HasCode> E getFromCode(Class<E> type, int code) {
|
||||
for (E candidate : type.getEnumConstants()) {
|
||||
if (candidate.getCode() == code) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
throw new IllegalArgumentException("Cannot create " + type.getName() + " from the code " + code);
|
||||
}
|
||||
|
||||
int getCode();
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.util;
|
||||
|
||||
import java.util.StringJoiner;
|
||||
|
||||
/**
|
||||
* debug util
|
||||
*/
|
||||
public class Printer {
|
||||
|
||||
public static String printHex(byte[] bytes) {
|
||||
StringJoiner s = new StringJoiner(" ");
|
||||
for (byte aByte : bytes) {
|
||||
s.add(leftpad(Long.toString(aByte & 0xFF, 16).toUpperCase()));
|
||||
}
|
||||
|
||||
return s.toString();
|
||||
}
|
||||
|
||||
private static String leftpad(String text) {
|
||||
return String.format("%1$2s", text).replace(' ', '0');
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.util;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* reads big-endian unsigned integers from bytebuffer
|
||||
*/
|
||||
public class UnsignedIntReader {
|
||||
public static long readU32(ByteBuffer buffer) {
|
||||
return buffer.getInt() & 0xFFFFFFFFL;
|
||||
}
|
||||
|
||||
public static int readU16(ByteBuffer buffer) {
|
||||
return buffer.getShort() & 0xFFFF;
|
||||
}
|
||||
|
||||
public static int readU8(ByteBuffer buffer) {
|
||||
return buffer.get() & 0xFF;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.validation;
|
||||
|
||||
// TODO use
|
||||
public class Array {
|
||||
public static Result compare(byte[] a1, byte[] a2) {
|
||||
if (a1 == null) {
|
||||
if (a2 == null) {
|
||||
return new Result(true, "both are null");
|
||||
} else {
|
||||
return new Result(false, "first is null, second is non-null");
|
||||
}
|
||||
} else {
|
||||
if (a2 == null) {
|
||||
return new Result(false, "first is non-null, second is null");
|
||||
}
|
||||
}
|
||||
if (a1.length != a2.length) {
|
||||
return new Result(false, "arrays do not have equal lengths: " + a1.length + " vs: " + a2.length);
|
||||
} else {
|
||||
for (int i = 0; i < a1.length; i++) {
|
||||
if (a1[i] != a2[i]) {
|
||||
return new Result(false, "arrays differ at index: " + i);
|
||||
}
|
||||
}
|
||||
return new Result(true, "Same");
|
||||
}
|
||||
}
|
||||
|
||||
static class Result {
|
||||
final boolean areEqual;
|
||||
final String userMessage;
|
||||
|
||||
Result(boolean areEqual, String userMessage) {
|
||||
this.areEqual = areEqual;
|
||||
this.userMessage = userMessage;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.validation;
|
||||
|
||||
public class ValidationException extends RuntimeException{
|
||||
|
||||
public ValidationException(String message){
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.validation;
|
||||
|
||||
public class Validator {
|
||||
public static void validate(String message, Object v1, Object v2) {
|
||||
if (!v1.equals(v2)) {
|
||||
throw new ValidationException(message + ": Not OK: " + v1 + " != " + v2);
|
||||
}
|
||||
}
|
||||
|
||||
//TODO use or remove
|
||||
public static void notNull(String message, Object value) {
|
||||
if (value == null) {
|
||||
throw new ValidationException("Not OK: " + message + ": is NULL");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package nl.sanderhautvast.sqlighter.fileviewer.util;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
|
||||
public class GetEncodingFromCodeTest {
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
assertSame(Encoding.UTF_8, Encoding.fromCode(1));
|
||||
assertSame(Encoding.UTF_16LE, Encoding.fromCode(2));
|
||||
assertSame(Encoding.UTF_16BE, Encoding.fromCode(3));
|
||||
}
|
||||
}
|
||||
8
notes.md
Normal file
8
notes.md
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
**compile sqlite amalgamation for debug**
|
||||
|
||||
`gcc shell.c sqlite3.c -lpthread -ldl -lm \
|
||||
-DSQLITE_DEBUG \
|
||||
-DSQLITE_ENABLE_EXPLAIN_COMMENTS \
|
||||
-DSQLITE_ENABLE_TREETRACE \
|
||||
-DSQLITE_ENABLE_WHERETRACE \
|
||||
-o sqlite3`
|
||||
50
pom.xml
Normal file
50
pom.xml
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>nl.sanderhautvast</groupId>
|
||||
<artifactId>sqlighter-pom</artifactId>
|
||||
<packaging>pom</packaging>
|
||||
<version>1.1.1</version>
|
||||
<modules>
|
||||
<module>sqlighter</module>
|
||||
<module>fileviewer</module>
|
||||
</modules>
|
||||
<name>sqlighter</name>
|
||||
<description>on the fly creation of sqlite native format from any database query</description>
|
||||
<properties>
|
||||
<java.version>11</java.version>
|
||||
</properties>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.8.0</version>
|
||||
<configuration>
|
||||
<release>11</release>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>gitlab-maven</id>
|
||||
<url>https://gitlab.com/api/v4/projects/40344554/packages/maven</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
|
||||
<distributionManagement>
|
||||
<repository>
|
||||
<id>gitlab-maven</id>
|
||||
<url>https://gitlab.com/api/v4/projects/40344554/packages/maven</url>
|
||||
</repository>
|
||||
|
||||
<snapshotRepository>
|
||||
<id>gitlab-maven</id>
|
||||
<url>https://gitlab.com/api/v4/projects/40344554/packages/maven</url>
|
||||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
</project>
|
||||
BIN
screenshot.png
Normal file
BIN
screenshot.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 314 KiB |
60
sqlighter/pom.xml
Normal file
60
sqlighter/pom.xml
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<groupId>nl.sanderhautvast</groupId>
|
||||
<artifactId>sqlighter-pom</artifactId>
|
||||
<version>1.1.1</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>sqlighter</artifactId>
|
||||
<version>1.1.1</version>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.xerial</groupId>
|
||||
<artifactId>sqlite-jdbc</artifactId>
|
||||
<version>3.39.3.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter-engine</artifactId>
|
||||
<version>5.9.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
<version>4.8.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-junit-jupiter</artifactId>
|
||||
<version>4.8.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>2.22.2</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-failsafe-plugin</artifactId>
|
||||
<version>2.22.2</version>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
package nl.sanderhautvast.sqlighter;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.page.Page;
|
||||
import nl.sanderhautvast.sqlighter.page.PageCacheFactory;
|
||||
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static nl.sanderhautvast.sqlighter.SQLiteConstants.*;
|
||||
|
||||
/**
|
||||
* Limited to one table. As this is the main use case, this will probably not change.
|
||||
* Please note that you can put whatever in it (does not have to reflect the actual source database structure),
|
||||
* including for example the result of a complex join
|
||||
*/
|
||||
public class Database {
|
||||
|
||||
public static short pageSize = 0x1000;
|
||||
|
||||
public static final String PAGESIZE_PROPERTY = "pagesize";
|
||||
|
||||
private final SchemaRecord schema;
|
||||
|
||||
final List<Page> leafPages;
|
||||
|
||||
private int pageCounter = 3;
|
||||
|
||||
/*
|
||||
* assumes 1 schema record ie 1 table. This might not change
|
||||
*/
|
||||
public Database(SchemaRecord schemaRecord, List<Page> leafPages) {
|
||||
this.schema = schemaRecord;
|
||||
this.leafPages = leafPages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the database as SQLite file to a file
|
||||
* @param filename name to write to
|
||||
* @throws IOException if underlying i/o raises error
|
||||
*/
|
||||
public void write(String filename) throws IOException {
|
||||
try (FileOutputStream outputStream = new FileOutputStream(filename)) {
|
||||
write(outputStream);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the database as SQLite file to the stream
|
||||
* @param outputStream any stream
|
||||
* @throws IOException if underlying i/o raises error
|
||||
*/
|
||||
public void write(OutputStream outputStream) throws IOException {
|
||||
List<? extends Page> currentTopLayer = this.leafPages;
|
||||
int nPages = currentTopLayer.size();
|
||||
while (currentTopLayer.size() > 1) { // interior page needed?
|
||||
currentTopLayer = createInteriorPages(currentTopLayer);
|
||||
nPages += currentTopLayer.size();
|
||||
}
|
||||
assert !currentTopLayer.isEmpty();
|
||||
Page tableRootPage = currentTopLayer.get(0); //
|
||||
outputStream.write(createHeaderPage(nPages + 1).getData());
|
||||
setChildReferencesAndWrite(tableRootPage, outputStream);
|
||||
outputStream.close();
|
||||
}
|
||||
|
||||
private void setChildReferencesAndWrite(Page page, OutputStream outputStream) {
|
||||
if (page.isInterior()) {
|
||||
setChildReferences(page);
|
||||
}
|
||||
write(page, outputStream);
|
||||
PageCacheFactory.getPageCache().release(page);
|
||||
//recurse
|
||||
for (Page child : page.getChildren()) {
|
||||
setChildReferencesAndWrite(child, outputStream);
|
||||
}
|
||||
}
|
||||
|
||||
private void write(Page page, OutputStream outputStream) {
|
||||
try {
|
||||
outputStream.write(page.getData());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private void setChildReferences(Page page) {
|
||||
page.setForwardPosition(Page.POSITION_CELL_COUNT);
|
||||
page.putU16(page.getChildren().size() - 1);
|
||||
|
||||
for (int i = 0; i < page.getChildren().size() - 1; i++) { // except right-most pointer
|
||||
page.setForwardPosition(Page.START + i * 2);
|
||||
int position = page.getU16(); // read the position that was written in an earlier pass
|
||||
page.setForwardPosition(position); // go to the cell at that location
|
||||
page.putU32(pageCounter++); // add page reference
|
||||
}
|
||||
page.setForwardPosition(Page.POSITION_RIGHTMOST_POINTER);
|
||||
page.putU32(pageCounter++);
|
||||
}
|
||||
|
||||
private Page createHeaderPage(int nPages) {
|
||||
Page headerPage = Page.newHeader(Database.pageSize);
|
||||
writeHeader(headerPage, nPages);
|
||||
int payloadLocationWriteLocation = headerPage.getForwardPosition(); // mark current position
|
||||
|
||||
int payloadLocation = writeSchema(headerPage, schema); //write schema payload from the end
|
||||
headerPage.setForwardPosition(payloadLocationWriteLocation); // go back to marked position
|
||||
headerPage.putU16(payloadLocation); //payload start
|
||||
headerPage.skipForward(1); // the number of fragmented free bytes within the cell content area
|
||||
headerPage.putU16(payloadLocation); // first cell
|
||||
return headerPage;
|
||||
}
|
||||
|
||||
private int writeSchema(Page rootPage, SchemaRecord schemaRecord) {
|
||||
rootPage.putBackward(schemaRecord.toRecord().toBytes());
|
||||
return rootPage.getBackwardPosition();
|
||||
}
|
||||
|
||||
private List<Page> createInteriorPages(List<? extends Page> childPages) {
|
||||
List<Page> interiorPages = new ArrayList<>();
|
||||
Page interiorPage = PageCacheFactory.getPageCache().getInteriorPage();
|
||||
interiorPage.setKey(childPages.stream().mapToLong(Page::getKey).max().orElse(-1));
|
||||
interiorPage.setForwardPosition(Page.START);
|
||||
int pageIndex;
|
||||
for (pageIndex = 0; pageIndex < childPages.size() - 1; pageIndex++) {
|
||||
Page leafPage = childPages.get(pageIndex);
|
||||
if (interiorPage.getBackwardPosition() < interiorPage.getForwardPosition() + 10) {
|
||||
interiorPage.setForwardPosition(Page.START_OF_CONTENT_AREA);
|
||||
interiorPage.putU16(interiorPage.getBackwardPosition());
|
||||
interiorPage.skipForward(5);
|
||||
|
||||
interiorPages.add(interiorPage);
|
||||
|
||||
interiorPage = PageCacheFactory.getPageCache().getInteriorPage();
|
||||
interiorPage.setForwardPosition(Page.START);
|
||||
}
|
||||
addCellWithPageRef(interiorPage, leafPage);
|
||||
interiorPage.addChild(leafPage);
|
||||
}
|
||||
|
||||
// write start of payload
|
||||
interiorPage.setForwardPosition(Page.START_OF_CONTENT_AREA);
|
||||
interiorPage.putU16(interiorPage.getBackwardPosition());
|
||||
interiorPage.skipForward(5);
|
||||
interiorPage.addChild(childPages.get(pageIndex));
|
||||
interiorPages.add(interiorPage);
|
||||
return interiorPages;
|
||||
}
|
||||
|
||||
private void addCellWithPageRef(Page interiorPage, Page leafPage) {
|
||||
byte[] keyAsBytes = Varint.write(leafPage.getKey());
|
||||
ByteBuffer cell = ByteBuffer.allocate(6 + keyAsBytes.length);
|
||||
cell.position(5);
|
||||
cell.put(keyAsBytes);
|
||||
|
||||
// write cell to page, starting at the end
|
||||
interiorPage.putBackward(cell.array());
|
||||
interiorPage.putU16(interiorPage.getBackwardPosition());
|
||||
}
|
||||
|
||||
private void writeHeader(Page rootpage, int nPages) {
|
||||
rootpage.putU8(MAGIC_HEADER);
|
||||
rootpage.putU16(rootpage.size());
|
||||
rootpage.putU8(FILE_FORMAT_WRITE_VERSION);
|
||||
rootpage.putU8(FILE_FORMAT_READ_VERSION);
|
||||
rootpage.putU8(RESERVED_SIZE);
|
||||
rootpage.putU8(MAX_EMBED_PAYLOAD_FRACTION);
|
||||
rootpage.putU8(MIN_EMBED_PAYLOAD_FRACTION);
|
||||
rootpage.putU8(LEAF_PAYLOAD_FRACTION);
|
||||
rootpage.putU32(FILECHANGE_COUNTER);
|
||||
rootpage.putU32(nPages);// file size in pages
|
||||
rootpage.putU32(FREELIST_TRUNK_PAGE_HUMBER);// Page number of the first freelist trunk page.
|
||||
rootpage.putU32(TOTAL_N_FREELIST_PAGES);
|
||||
rootpage.putU32(SCHEMA_COOKIE);
|
||||
rootpage.putU32(SQLITE_SCHEMAVERSION);
|
||||
rootpage.putU32(SUGGESTED_CACHESIZE);
|
||||
rootpage.putU32(LARGEST_ROOT_BTREE_PAGE);
|
||||
rootpage.putU32(ENCODING_UTF8);
|
||||
rootpage.putU32(USER_VERSION);
|
||||
rootpage.putU32(VACUUM_MODE_OFF);// True (non-zero) for incremental-vacuum mode. False (zero) otherwise.
|
||||
rootpage.putU32(APP_ID);// Application ID
|
||||
rootpage.putU8(FILLER);// Reserved for expansion. Must be zero.
|
||||
rootpage.putU8(VERSION_VALID_FOR);// The version-valid-for number
|
||||
rootpage.putU8(SQLITE_VERSION);// SQLITE_VERSION_NUMBER
|
||||
rootpage.putU8(SQLiteConstants.TABLE_LEAF_PAGE); // leaf table b-tree page for schema
|
||||
rootpage.putU16(NO_FREE_BLOCKS); // zero if there are no freeblocks
|
||||
rootpage.putU16(1); // the number of cells on the page
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
package nl.sanderhautvast.sqlighter;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.data.LtRecord;
|
||||
import nl.sanderhautvast.sqlighter.page.Page;
|
||||
import nl.sanderhautvast.sqlighter.page.PageCacheFactory;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* The database builder is the main interface to create a database.
|
||||
*/
|
||||
public class DatabaseBuilder {
|
||||
|
||||
private final List<Page> leafPages = new ArrayList<>();
|
||||
private Page currentPage;
|
||||
|
||||
private SchemaRecord schemaRecord;
|
||||
|
||||
private int nRecordsOnCurrentPage;
|
||||
|
||||
public DatabaseBuilder() {
|
||||
try {
|
||||
Database.pageSize = Short.parseShort(System.getProperty(Database.PAGESIZE_PROPERTY));
|
||||
} catch (NumberFormatException ex) {
|
||||
//ignore
|
||||
}
|
||||
|
||||
createPage();
|
||||
}
|
||||
|
||||
public void addRecord(final LtRecord record) {
|
||||
byte[] recordBytes = record.toBytes();
|
||||
if (!newRecordFits(recordBytes)) {
|
||||
finishCurrentPage();
|
||||
createPage();
|
||||
}
|
||||
currentPage.setKey(record.getRowId()); //gets updated until page is finished
|
||||
currentPage.putBackward(recordBytes);
|
||||
currentPage.putU16(currentPage.getBackwardPosition());
|
||||
nRecordsOnCurrentPage += 1;
|
||||
}
|
||||
|
||||
public void addSchema(String tableName, String ddl) {
|
||||
this.schemaRecord = new SchemaRecord(1, tableName, 2, ddl);
|
||||
}
|
||||
|
||||
public Database build() {
|
||||
currentPage.setForwardPosition(Page.POSITION_CELL_COUNT);
|
||||
currentPage.putU16(nRecordsOnCurrentPage);
|
||||
|
||||
if (nRecordsOnCurrentPage > 0) {
|
||||
currentPage.putU16(currentPage.getBackwardPosition());
|
||||
} else {
|
||||
currentPage.putU16(currentPage.getBackwardPosition() - 1);
|
||||
}
|
||||
|
||||
return new Database(schemaRecord, leafPages);
|
||||
}
|
||||
|
||||
private boolean newRecordFits(byte[] newBytes) {
|
||||
return currentPage.getBackwardPosition() - newBytes.length -2 > currentPage.getForwardPosition();
|
||||
// 2 for cell pointer length
|
||||
}
|
||||
|
||||
private void finishCurrentPage() {
|
||||
currentPage.setForwardPosition(Page.POSITION_CELL_COUNT);
|
||||
currentPage.putU16(nRecordsOnCurrentPage);
|
||||
currentPage.putU16(currentPage.getBackwardPosition());
|
||||
}
|
||||
|
||||
private void createPage() {
|
||||
currentPage = PageCacheFactory.getPageCache().getLeafPage();
|
||||
currentPage.setForwardPosition(8);
|
||||
leafPages.add(currentPage);
|
||||
nRecordsOnCurrentPage = 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
package nl.sanderhautvast.sqlighter;
|
||||
|
||||
/**
|
||||
* special values for SQLite.
|
||||
*
|
||||
* See <a href="https://sqlite.org/fileformat2.html">Database File Format </a>
|
||||
*/
|
||||
public class SQLiteConstants {
|
||||
public static final byte[] MAGIC_HEADER = new byte[]{0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00};
|
||||
|
||||
public static final byte FILE_FORMAT_WRITE_VERSION = 1; // legacy
|
||||
public static final byte FILE_FORMAT_READ_VERSION = 1; // legacy
|
||||
public static final byte RESERVED_SIZE = 0;
|
||||
public static final byte MAX_EMBED_PAYLOAD_FRACTION = 0x40;
|
||||
public static final byte MIN_EMBED_PAYLOAD_FRACTION = 0x20;
|
||||
public static final byte LEAF_PAYLOAD_FRACTION = 0x20;
|
||||
public static final int FILECHANGE_COUNTER = 1;
|
||||
public static final int FREELIST_TRUNK_PAGE_HUMBER = 0;
|
||||
public static final int TOTAL_N_FREELIST_PAGES = 0;
|
||||
public static final int SCHEMA_COOKIE = 1;
|
||||
public static final int SQLITE_SCHEMAVERSION = 4;
|
||||
public static final int SUGGESTED_CACHESIZE = 0;
|
||||
public static final int LARGEST_ROOT_BTREE_PAGE = 0; // zero when not in auto-vacuum mode
|
||||
public static final int ENCODING_UTF8 = 1; // The database text encoding. A value of 1 means UTF-8. A value of 2 means UTF-16le. A value of 3 means UTF-16be.
|
||||
public static final int USER_VERSION = 0;
|
||||
public static final int VACUUM_MODE_OFF = 0; // not used
|
||||
public static final int APP_ID = 0;
|
||||
public static final byte[] FILLER = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
|
||||
}; // 20 bytes for some future use
|
||||
public static final byte[] VERSION_VALID_FOR = {0, 0, 0x03, -123};
|
||||
public static final byte[] SQLITE_VERSION = {0x00, 0x2e, 0x5F, 0x1A};
|
||||
public static final short NO_FREE_BLOCKS = 0;
|
||||
public static final byte TABLE_LEAF_PAGE = 0x0d; //TODO enum?
|
||||
public static final byte TABLE_INTERIOR_PAGE = 0x05;
|
||||
public static final byte INDEX_LEAF_PAGE = 0x0a;
|
||||
public static final byte INDEX_INTERIOR_PAGE = 0x02;
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package nl.sanderhautvast.sqlighter;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.data.LtRecord;
|
||||
import nl.sanderhautvast.sqlighter.data.LtValue;
|
||||
|
||||
/*
|
||||
* Is a record in the sqlites_schema table
|
||||
* and a special case of a Record
|
||||
* class is being used for both reading and writing
|
||||
*
|
||||
*/
|
||||
public class SchemaRecord {
|
||||
|
||||
private final long rowid;
|
||||
private final String tableName;
|
||||
private final long rootpage;
|
||||
private final String sql;
|
||||
|
||||
public SchemaRecord(long rowid, String tableName, long rootpage, String sql) {
|
||||
this.rowid = rowid;
|
||||
this.tableName = tableName;
|
||||
this.rootpage = rootpage;
|
||||
this.sql = sql;
|
||||
}
|
||||
|
||||
public String getTableName() {
|
||||
return tableName;
|
||||
}
|
||||
|
||||
public long getRootpage() {
|
||||
return rootpage;
|
||||
}
|
||||
|
||||
public String getSql() {
|
||||
return sql;
|
||||
}
|
||||
|
||||
public LtRecord toRecord(){
|
||||
LtRecord record = new LtRecord(rowid);
|
||||
record.addValue(LtValue.of("table"));
|
||||
record.addValue(LtValue.of(getTableName().toLowerCase()));
|
||||
record.addValue(LtValue.of(getTableName().toLowerCase()));
|
||||
record.addValue(LtValue.of(getRootpage()));
|
||||
record.addValue(LtValue.of(getSql()));
|
||||
return record;
|
||||
}
|
||||
}
|
||||
156
sqlighter/src/main/java/nl/sanderhautvast/sqlighter/Varint.java
Normal file
156
sqlighter/src/main/java/nl/sanderhautvast/sqlighter/Varint.java
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
package nl.sanderhautvast.sqlighter;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* Writes integers to byte representation like Sqlite's putVarint64
|
||||
* not threadsafe (take out B8 and B9 if you need that)
|
||||
*/
|
||||
public final class Varint {
|
||||
|
||||
private Varint() {
|
||||
}
|
||||
|
||||
public static byte[] write(long v) {
|
||||
if ((v & ((0xff000000L) << 32)) != 0) {
|
||||
byte[] result = new byte[9];
|
||||
result[8] = (byte) v;
|
||||
v >>= 8;
|
||||
for (int i = 7; i >= 0; i--) {
|
||||
result[i] = (byte) ((v & 0x7f) | 0x80);
|
||||
v >>= 7;
|
||||
}
|
||||
return result;
|
||||
} else {
|
||||
int n;
|
||||
byte[] buf = new byte[8];
|
||||
for (n = 0; v != 0; n++, v >>= 7) {
|
||||
buf[n] = (byte) ((v & 0x7f) | 0x80);
|
||||
}
|
||||
buf[0] &= 0x7f;
|
||||
byte[] result = new byte[n];
|
||||
for (int i = 0, j = n - 1; j >= 0; j--, i++) {
|
||||
result[i] = buf[j];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* read a long value from a variable nr of bytes in varint format
|
||||
* NB the end is encoded in the bytes, and the passed byte array may be bigger, but the
|
||||
* remainder is not read. It's up to the caller to do it right.
|
||||
*/
|
||||
public static long read(byte[] bytes) {
|
||||
return read(ByteBuffer.wrap(bytes));
|
||||
}
|
||||
|
||||
/*
|
||||
* read a long value from a variable nr of bytes in varint format
|
||||
*
|
||||
* copied from the sqlite source, with some java specifics, most notably the addition of
|
||||
* &0xFF for the right conversion from byte => signed in java, but to be interpreted as unsigned,
|
||||
* to long
|
||||
*
|
||||
* Does not have the issue that the read(byte[] bytes) method has. The nr of bytes read is determined
|
||||
* by the varint64 format.
|
||||
*
|
||||
* TODO write specialized version for u32
|
||||
*/
|
||||
public static long read(ByteBuffer buffer) {
|
||||
int SLOT_2_0 = 0x001fc07f;
|
||||
int SLOT_4_2_0 = 0xf01fc07f;
|
||||
|
||||
long a = buffer.get() & 0xFF;
|
||||
if ((a & 0x80) == 0) {
|
||||
return a;
|
||||
}
|
||||
|
||||
long b = buffer.get() & 0xFF;
|
||||
if ((b & 0x80) == 0) {
|
||||
a &= 0x7F;
|
||||
a = a << 7;
|
||||
a |= b;
|
||||
return a;
|
||||
}
|
||||
|
||||
a = a << 14;
|
||||
a |= (buffer.get() & 0xFF);
|
||||
if ((a & 0x80) == 0) {
|
||||
a &= SLOT_2_0;
|
||||
b &= 0x7F;
|
||||
b = b << 7;
|
||||
a |= b;
|
||||
return a;
|
||||
}
|
||||
|
||||
a &= SLOT_2_0;
|
||||
b = b << 14;
|
||||
b |= (buffer.get() & 0xFF);
|
||||
if ((b & 0x80) == 0) {
|
||||
b &= SLOT_2_0;
|
||||
a = a << 7;
|
||||
a |= b;
|
||||
return a;
|
||||
}
|
||||
|
||||
b &= SLOT_2_0;
|
||||
long s = a;
|
||||
a = a << 14;
|
||||
int m = buffer.get() & 0xFF;
|
||||
a |= m;
|
||||
if ((a & 0x80) == 0) {
|
||||
b = b << 7;
|
||||
a |= b;
|
||||
s = s >> 18;
|
||||
return (s << 32) | a;
|
||||
}
|
||||
|
||||
s = s << 7;
|
||||
s |= b;
|
||||
b = b << 14;
|
||||
b |= (buffer.get() & 0xFF);
|
||||
if ((b & 0x80) == 0) {
|
||||
a &= SLOT_2_0;
|
||||
a = a << 7;
|
||||
a |= b;
|
||||
s = s >> 18;
|
||||
return (s << 32) | a;
|
||||
}
|
||||
|
||||
a = a << 14;
|
||||
a |= (buffer.get() & 0xFF);
|
||||
if ((a & 0x80) == 0) {
|
||||
a &= SLOT_4_2_0;
|
||||
b &= SLOT_2_0;
|
||||
b = b << 7;
|
||||
a |= b;
|
||||
s = s >> 11;
|
||||
return (s << 32) | a;
|
||||
}
|
||||
|
||||
a &= SLOT_2_0;
|
||||
b = b << 14;
|
||||
b |= (buffer.get() & 0xFF);
|
||||
if ((b & 0x80) == 0) {
|
||||
b &= SLOT_4_2_0;
|
||||
a = a << 7;
|
||||
a |= b;
|
||||
s = s >> 4;
|
||||
return (s << 32) | a;
|
||||
}
|
||||
|
||||
a = a << 15;
|
||||
a |= (buffer.get() & 0xFF);
|
||||
b &= SLOT_2_0;
|
||||
|
||||
b = b << 8;
|
||||
a |= b;
|
||||
s = s << 4;
|
||||
b = m;
|
||||
b &= 0x7F;
|
||||
b = b >> 3;
|
||||
s |= b;
|
||||
return (s << 32) | a;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
package nl.sanderhautvast.sqlighter.data;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.Varint;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Record in sqlite database.
|
||||
* Used for reading and writing.
|
||||
*/
|
||||
public final class LtRecord {
|
||||
|
||||
private final long rowId;
|
||||
|
||||
private int dataTypesLength = -1;
|
||||
private int valuesLength = -1;
|
||||
|
||||
private final List<LtValue> values = new ArrayList<>(10);
|
||||
|
||||
public LtRecord(long rowId) {
|
||||
this.rowId = rowId;
|
||||
}
|
||||
|
||||
public void addValues(LtValue... values) {
|
||||
this.values.addAll(Arrays.asList(values));
|
||||
}
|
||||
|
||||
public void addValue(LtValue value) {
|
||||
this.values.add(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* write the record to an array of bytes
|
||||
*/
|
||||
public byte[] toBytes() {
|
||||
int dataLength = getDataLength();
|
||||
byte[] lengthBytes = Varint.write(dataLength);
|
||||
byte[] rowIdBytes = Varint.write(rowId);
|
||||
|
||||
ByteBuffer buffer = ByteBuffer.allocate(lengthBytes.length + rowIdBytes.length + dataLength);
|
||||
buffer.put(lengthBytes);
|
||||
buffer.put(rowIdBytes);
|
||||
|
||||
buffer.put(Varint.write(dataTypesLength));
|
||||
|
||||
//types
|
||||
for (LtValue value : values) {
|
||||
value.writeType(buffer);
|
||||
}
|
||||
|
||||
//values
|
||||
for (LtValue value : values) {
|
||||
value.writeValue(buffer);
|
||||
}
|
||||
|
||||
return buffer.array();
|
||||
}
|
||||
|
||||
public int getDataLength() {
|
||||
if (dataTypesLength < 0 || valuesLength < 0) {
|
||||
dataTypesLength = 1;
|
||||
valuesLength = 0;
|
||||
for (LtValue value : values) {
|
||||
dataTypesLength += value.getDataTypeLength();
|
||||
valuesLength += value.getValueLength();
|
||||
}
|
||||
}
|
||||
return dataTypesLength + valuesLength;
|
||||
}
|
||||
|
||||
public long getRowId() {
|
||||
return rowId;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
public List<LtValue> getValues() {
|
||||
return values;
|
||||
}
|
||||
|
||||
/**
|
||||
* returns the value at the specified column index (0 based)
|
||||
*/
|
||||
public LtValue getValue(int column) {
|
||||
return values.get(column);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
LtRecord record = (LtRecord) o;
|
||||
return rowId == record.rowId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(rowId);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,127 @@
|
|||
package nl.sanderhautvast.sqlighter.data;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.Varint;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/*
|
||||
* NB Value classes derive their equality from their identity. I.e. no equals/hashcode
|
||||
*/
|
||||
public class LtValue {
|
||||
private static final byte FLOAT_TYPE = 7;
|
||||
private static final int STRING_OFFSET = 13;
|
||||
private static final int BYTES_OFFSET = 12;
|
||||
|
||||
private final byte[] type;
|
||||
private final byte[] value;
|
||||
|
||||
protected LtValue(byte[] type, byte[] value) {
|
||||
this.type = type;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the length of serialType + the length of the value
|
||||
*/
|
||||
public int getValueLength() {
|
||||
return value.length;
|
||||
}
|
||||
|
||||
public int getDataTypeLength() {
|
||||
return type.length;
|
||||
}
|
||||
|
||||
public void writeType(ByteBuffer buffer) {
|
||||
buffer.put(type);
|
||||
}
|
||||
|
||||
public void writeValue(ByteBuffer buffer) {
|
||||
buffer.put(value);
|
||||
}
|
||||
|
||||
public byte[] getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
|
||||
public static LtValue of(String value) {
|
||||
return new LtValue(Varint.write(value == null ? 0 : ((long) (value.getBytes(StandardCharsets.UTF_8).length) << 1) + STRING_OFFSET),
|
||||
value == null ? new byte[0] : value.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
public static LtValue of(long value) {
|
||||
byte[] valueAsBytes = getValueAsBytes(value);
|
||||
return new LtValue(getIntegerType(value, valueAsBytes.length), valueAsBytes);
|
||||
}
|
||||
|
||||
public static LtValue of(double value) {
|
||||
return new LtValue(new byte[]{FLOAT_TYPE}, ByteBuffer.wrap(new byte[8]).putDouble(0, value).array());
|
||||
}
|
||||
|
||||
public static LtValue of(byte[] value) {
|
||||
return new LtValue(Varint.write(((long) value.length << 1) + BYTES_OFFSET), value);
|
||||
}
|
||||
|
||||
public static byte[] getIntegerType(long value, int bytesLength) {
|
||||
if (value == 0) {
|
||||
return new byte[]{8};
|
||||
} else if (value == 1) {
|
||||
return new byte[]{9};
|
||||
} else {
|
||||
if (bytesLength < 5) {
|
||||
return Varint.write(bytesLength);
|
||||
} else if (bytesLength < 7) {
|
||||
return Varint.write(5);
|
||||
} else return Varint.write(6);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* static because it's used in the constructor
|
||||
*/
|
||||
public static byte[] getValueAsBytes(long value) {
|
||||
if (value == 0) {
|
||||
return new byte[0];
|
||||
} else if (value == 1) {
|
||||
return new byte[0];
|
||||
} else {
|
||||
return longToBytes(value, getLengthOfByteEncoding(value));
|
||||
}
|
||||
}
|
||||
|
||||
public static int getLengthOfByteEncoding(long value) {
|
||||
long u;
|
||||
if (value < 0) {
|
||||
u = ~value;
|
||||
} else {
|
||||
u = value;
|
||||
}
|
||||
if (u <= 127) {
|
||||
return 1;
|
||||
} else if (u <= 32767) {
|
||||
return 2;
|
||||
} else if (u <= 8388607) {
|
||||
return 3;
|
||||
} else if (u <= 2147483647) {
|
||||
return 4;
|
||||
} else if (u <= 140737488355327L) {
|
||||
return 6;
|
||||
} else {
|
||||
return 8;
|
||||
}
|
||||
}
|
||||
|
||||
public static byte[] longToBytes(long n, int nbytes) {
|
||||
byte[] b = new byte[nbytes];
|
||||
for (int i = 0; i < nbytes; i++) {
|
||||
b[i] = (byte) ((n >> (nbytes - i - 1) * 8) & 0xFF);
|
||||
}
|
||||
|
||||
return b;
|
||||
}
|
||||
|
||||
public byte[] getType() {
|
||||
return type;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
package nl.sanderhautvast.sqlighter.jdbc;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.data.LtRecord;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.ResultSetMetaData;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Use this class to generate an SQLite database from your JDBC results
|
||||
*/
|
||||
public class ResulSet2SQLite {
|
||||
|
||||
private final ValueMapper valueMapper;
|
||||
private final int columnCount;
|
||||
|
||||
private long rowid = 1;
|
||||
|
||||
public ResulSet2SQLite(ResultSetMetaData metadata) throws SQLException {
|
||||
valueMapper = new ValueMapper(getSqlTypesFromJdbcResult(metadata));
|
||||
|
||||
this.columnCount = getColumnCount(metadata);
|
||||
}
|
||||
|
||||
public LtRecord mapRow(ResultSet result) throws SQLException {
|
||||
LtRecord record = new LtRecord(rowid++);
|
||||
for (int columnIndex = 1; columnIndex <= columnCount; columnIndex++) {
|
||||
valueMapper.addValueFromJdbcResult(result, record, columnIndex);
|
||||
}
|
||||
|
||||
return record;
|
||||
}
|
||||
|
||||
static List<Integer> getSqlTypesFromJdbcResult(ResultSetMetaData metaData) throws SQLException {
|
||||
List<Integer> types = new ArrayList<>();
|
||||
for (int i = 1; i <= getColumnCount(metaData); i++) {
|
||||
types.add(metaData.getColumnType(i));
|
||||
}
|
||||
return types;
|
||||
}
|
||||
|
||||
private static int getColumnCount(ResultSetMetaData metaData) throws SQLException {
|
||||
return metaData.getColumnCount();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
package nl.sanderhautvast.sqlighter.jdbc;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.data.LtRecord;
|
||||
import nl.sanderhautvast.sqlighter.data.LtValue;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Types;
|
||||
import java.util.List;
|
||||
|
||||
class ValueMapper {
|
||||
|
||||
private final List<Integer> types;
|
||||
|
||||
public ValueMapper(List<Integer> types) {
|
||||
this.types = types;
|
||||
}
|
||||
|
||||
void addValueFromJdbcResult(ResultSet result, LtRecord record, int columnIndex) throws SQLException {
|
||||
switch (types.get(columnIndex - 1)) { // index in metadata starts with 1, index in list starts with 0
|
||||
case Types.BLOB:
|
||||
case Types.CLOB: // handle as string?
|
||||
case Types.NCLOB: // handle as string?
|
||||
case Types.VARBINARY:
|
||||
case Types.LONGVARBINARY:
|
||||
case Types.JAVA_OBJECT:
|
||||
record.addValue(LtValue.of(result.getBytes(columnIndex)));
|
||||
break;
|
||||
case Types.NUMERIC:
|
||||
case Types.BIGINT:
|
||||
case Types.INTEGER:
|
||||
case Types.SMALLINT:
|
||||
case Types.DECIMAL:
|
||||
case Types.TINYINT:
|
||||
record.addValue(LtValue.of(result.getLong(columnIndex)));
|
||||
break;
|
||||
case Types.BOOLEAN:
|
||||
case Types.BIT: //?
|
||||
record.addValue(LtValue.of(result.getBoolean(columnIndex) ? 1 : 0));
|
||||
break;
|
||||
case Types.DATE:
|
||||
case Types.TIME:
|
||||
case Types.TIME_WITH_TIMEZONE:
|
||||
case Types.TIMESTAMP:
|
||||
case Types.TIMESTAMP_WITH_TIMEZONE:
|
||||
record.addValue(LtValue.of(result.getDate(columnIndex).getTime()));
|
||||
break;
|
||||
case Types.FLOAT:
|
||||
case Types.DOUBLE:
|
||||
record.addValue(LtValue.of(result.getDouble(columnIndex)));
|
||||
break;
|
||||
case Types.CHAR:
|
||||
case Types.NCHAR:
|
||||
case Types.VARCHAR:
|
||||
case Types.NVARCHAR:
|
||||
case Types.LONGVARCHAR:
|
||||
case Types.LONGNVARCHAR:
|
||||
record.addValue(LtValue.of(result.getString(columnIndex)));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
package nl.sanderhautvast.sqlighter.page;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.Database;
|
||||
import nl.sanderhautvast.sqlighter.SQLiteConstants;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Represents a SQLite page
|
||||
*/
|
||||
public final class Page {
|
||||
|
||||
public static int POSITION_RIGHTMOST_POINTER = 8; // first position after page header
|
||||
public static int START = 12; // first position after page header
|
||||
|
||||
public static final int POSITION_CELL_COUNT = 3;
|
||||
public static final int START_OF_CONTENT_AREA = 5;
|
||||
|
||||
private final byte[] data;
|
||||
|
||||
private long key;
|
||||
|
||||
private final List<Page> children = new ArrayList<>();
|
||||
|
||||
private int forwardPosition;
|
||||
private int backwardPosition;
|
||||
|
||||
private final PageType type;
|
||||
|
||||
static Page newLeaf() {
|
||||
return newLeaf(Database.pageSize);
|
||||
}
|
||||
|
||||
static Page newInterior() {
|
||||
return newInterior(Database.pageSize);
|
||||
}
|
||||
|
||||
static Page newLeaf(short pageSize) {
|
||||
Page page = new Page(PageType.TABLE_LEAF, pageSize);
|
||||
page.putU8(SQLiteConstants.TABLE_LEAF_PAGE);
|
||||
page.skipForward(2);
|
||||
return page;
|
||||
}
|
||||
|
||||
static Page newInterior(short pageSize) {
|
||||
Page page = new Page(PageType.TABLE_INTERIOR, pageSize);
|
||||
page.putU8(SQLiteConstants.TABLE_INTERIOR_PAGE);
|
||||
return page;
|
||||
}
|
||||
|
||||
|
||||
public static Page newHeader(int size) {
|
||||
return new Page(PageType.HEADER, size);
|
||||
}
|
||||
|
||||
public void addChild(Page child) {
|
||||
children.add(child);
|
||||
}
|
||||
|
||||
private Page(PageType type, int size) {
|
||||
this.type = type;
|
||||
data = new byte[size];
|
||||
forwardPosition = 0;
|
||||
backwardPosition = size;
|
||||
}
|
||||
|
||||
public int getForwardPosition() {
|
||||
return forwardPosition;
|
||||
}
|
||||
|
||||
public int getBackwardPosition() {
|
||||
return backwardPosition;
|
||||
}
|
||||
|
||||
public void setForwardPosition(int forwardPosition) {
|
||||
this.forwardPosition = forwardPosition;
|
||||
}
|
||||
|
||||
public void putU16(int value) {
|
||||
data[forwardPosition] = (byte) ((value >> 8) & 0xFF);
|
||||
data[forwardPosition + 1] = (byte) (value & 0xFF);
|
||||
forwardPosition += 2;
|
||||
}
|
||||
|
||||
public void putU32(long value) {
|
||||
data[forwardPosition] = (byte) ((value >> 24) & 0xFF);
|
||||
data[forwardPosition + 1] = (byte) ((value >> 16) & 0xFF);
|
||||
data[forwardPosition + 2] = (byte) ((value >> 8) & 0xFF);
|
||||
data[forwardPosition + 3] = (byte) (value & 0xFF);
|
||||
forwardPosition += 4;
|
||||
}
|
||||
|
||||
public void putU8(int value) {
|
||||
data[forwardPosition] = (byte) (value & 0xFF);
|
||||
forwardPosition += 1;
|
||||
}
|
||||
|
||||
public void putU8(byte[] value) {
|
||||
System.arraycopy(value, 0, data, forwardPosition, value.length);
|
||||
forwardPosition += value.length;
|
||||
}
|
||||
|
||||
public int getU16() {
|
||||
return ((data[forwardPosition] & 0xFF) << 8) + (data[forwardPosition + 1] & 0xFF);
|
||||
}
|
||||
|
||||
public void putBackward(byte[] value) {
|
||||
backwardPosition -= value.length;
|
||||
System.arraycopy(value, 0, data, backwardPosition, value.length);
|
||||
}
|
||||
|
||||
public void setKey(long key) {
|
||||
this.key = key;
|
||||
}
|
||||
|
||||
public long getKey() {
|
||||
return key;
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return data.length;
|
||||
}
|
||||
|
||||
public List<Page> getChildren() {
|
||||
return children;
|
||||
}
|
||||
|
||||
public byte[] getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
public void skipForward(int length) {
|
||||
this.forwardPosition += length;
|
||||
}
|
||||
|
||||
public boolean isInterior() {
|
||||
return type == PageType.TABLE_INTERIOR;
|
||||
}
|
||||
|
||||
public PageType getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
void reset() {
|
||||
this.forwardPosition = 0;
|
||||
this.backwardPosition = Database.pageSize;
|
||||
this.children.clear();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
package nl.sanderhautvast.sqlighter.page;
|
||||
|
||||
import java.util.Queue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
public class PageCache {
|
||||
|
||||
protected final Queue<Page> leafPages = new LinkedBlockingQueue<>();
|
||||
protected final Queue<Page> interiorPages = new LinkedBlockingQueue<>();
|
||||
|
||||
public Page getInteriorPage() {
|
||||
Page page = interiorPages.poll();
|
||||
if (page == null) {
|
||||
page = Page.newInterior();
|
||||
} else {
|
||||
page.reset();
|
||||
}
|
||||
return page;
|
||||
}
|
||||
|
||||
public Page getLeafPage() {
|
||||
Page page = leafPages.poll();
|
||||
if (page == null) {
|
||||
page = Page.newLeaf();
|
||||
} else {
|
||||
page.reset();
|
||||
}
|
||||
return page;
|
||||
}
|
||||
|
||||
public void release(Page page) {
|
||||
if (page.getType() == PageType.TABLE_INTERIOR) {
|
||||
interiorPages.add(page);
|
||||
} else if (page.getType() == PageType.TABLE_LEAF) {
|
||||
leafPages.add(page);
|
||||
}
|
||||
}
|
||||
|
||||
public void clear(){
|
||||
interiorPages.clear();
|
||||
leafPages.clear();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package nl.sanderhautvast.sqlighter.page;
|
||||
|
||||
import java.lang.ref.SoftReference;
|
||||
import java.util.Optional;
|
||||
|
||||
public class PageCacheFactory {
|
||||
|
||||
private static final ThreadLocal<SoftReference<PageCache>> threadlocalPageCache = new ThreadLocal<>();
|
||||
|
||||
public static PageCache getPageCache() {
|
||||
return Optional.ofNullable(threadlocalPageCache.get())
|
||||
.map(SoftReference::get)
|
||||
.orElseGet(() -> {
|
||||
PageCache pageCache = new PageCache();
|
||||
threadlocalPageCache.set(new SoftReference<>(pageCache));
|
||||
return pageCache;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
package nl.sanderhautvast.sqlighter.page;
|
||||
|
||||
public enum PageType {
|
||||
TABLE_LEAF,
|
||||
TABLE_INTERIOR,
|
||||
INDEX_LEAF,
|
||||
INDEX_INTERIOR,
|
||||
HEADER,
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
package nl.sanderhautvast.sqlighter;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.data.LtRecord;
|
||||
import nl.sanderhautvast.sqlighter.data.LtValue;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
|
||||
public class ControlTest {
|
||||
public static void main(String[] args) {
|
||||
System.out.println(Long.toString(520 * 4096L, 16));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void createDatabaseUsingSqliteJdbc() throws SQLException {
|
||||
try (Connection connection = DriverManager.getConnection("jdbc:sqlite:lite.db")) {
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
statement.executeUpdate("drop table if exists foo");
|
||||
}
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
statement.executeUpdate("create table foo (bar integer, baz varchar(1002))");
|
||||
String filler = "Hhgtttg".repeat(167);
|
||||
|
||||
for (int i = 0; i < 2000; i++) {
|
||||
statement.executeUpdate("insert into foo (bar, baz) values (42, '" + filler + "')");
|
||||
}
|
||||
}
|
||||
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
statement.executeUpdate("vacuum");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void createDatabaseUsingSqlighter() throws IOException {
|
||||
DatabaseBuilder databaseBuilder = new DatabaseBuilder();
|
||||
databaseBuilder.addSchema("foo",
|
||||
"CREATE TABLE foo(bar integer,baz varchar(1002))");
|
||||
|
||||
for (int i = 0; i < 2000; i++) {
|
||||
LtRecord record = new LtRecord(i);
|
||||
record.addValues(LtValue.of(42), LtValue.of("Hhgtttg".repeat(167)));
|
||||
databaseBuilder.addRecord(record);
|
||||
}
|
||||
|
||||
databaseBuilder.build().write("light.db");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
package nl.sanderhautvast.sqlighter;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.data.LtRecord;
|
||||
import nl.sanderhautvast.sqlighter.data.LtValue;
|
||||
import nl.sanderhautvast.sqlighter.page.Page;
|
||||
import nl.sanderhautvast.sqlighter.page.PageCacheFactory;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
|
||||
class LeafDbPageTest {
|
||||
|
||||
@BeforeEach
|
||||
public void setup() {
|
||||
Database.pageSize=64;
|
||||
PageCacheFactory.getPageCache().clear();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void teardown() {
|
||||
PageCacheFactory.getPageCache().clear();
|
||||
Database.pageSize=0x1000;
|
||||
}
|
||||
|
||||
@Test
|
||||
void testWriteDataToLeafPage() {
|
||||
LtRecord record1 = new LtRecord(1);
|
||||
record1.addValues(LtValue.of("hello"));
|
||||
LtRecord record2 = new LtRecord(2);
|
||||
record2.addValues(LtValue.of("world"));
|
||||
DatabaseBuilder databaseBuilder = new DatabaseBuilder();
|
||||
databaseBuilder.addRecord(record1);
|
||||
databaseBuilder.addRecord(record2);
|
||||
|
||||
List<Page> leafPages = databaseBuilder.build().leafPages;
|
||||
|
||||
assertFalse(leafPages.isEmpty());
|
||||
assertArrayEquals(new byte[]{
|
||||
0x0D, 0x00, 0x00, 0x00, 0x02, 0x00, 0x2E, 0x00, 0x00, 0x37, 0x00, 0x2e, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x02,
|
||||
0x02, 0x17, 0x77, 0x6F, 0x72, 0x6C, 0x64, 0x07, 0x01, 0x02, 0x17, 0x68, 0x65, 0x6C, 0x6C, 0x6F},
|
||||
leafPages.get(0).getData()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
package nl.sanderhautvast.sqlighter;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.data.LtRecord;
|
||||
import nl.sanderhautvast.sqlighter.data.LtValue;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class LtRecordTest {
|
||||
|
||||
@Test
|
||||
void testStringValue() {
|
||||
LtRecord record = new LtRecord(1);
|
||||
record.addValue(LtValue.of("string"));
|
||||
assertArrayEquals(new byte[]{8, 1, 2, 25, 115, 116, 114, 105, 110, 103}, record.toBytes());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
void testIntValue() {
|
||||
LtRecord record = new LtRecord(1);
|
||||
record.addValue(LtValue.of(2));
|
||||
assertArrayEquals(new byte[]{3, 1, 2, 1, 2}, record.toBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIntValue_0() {
|
||||
LtRecord record = new LtRecord(1);
|
||||
record.addValue(LtValue.of(0));
|
||||
assertArrayEquals(new byte[]{2, 1, 2, 8}, record.toBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFloatValue() {
|
||||
LtRecord record = new LtRecord(1);
|
||||
record.addValue(LtValue.of(2.0));
|
||||
assertArrayEquals(new byte[]{10, 1, 2, 7, 64, 0, 0, 0, 0, 0, 0, 0}, record.toBytes());
|
||||
assertEquals(10,record.getDataLength());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testStringIntValues() {
|
||||
LtRecord record = new LtRecord(1);
|
||||
record.addValue(LtValue.of("string"));
|
||||
record.addValue(LtValue.of(2));
|
||||
assertArrayEquals(new byte[]{10, 1, 3, 25, 1, 115, 116, 114, 105, 110, 103, 2}, record.toBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testStringInt_1Values() {
|
||||
LtRecord record = new LtRecord(1);
|
||||
record.addValue(LtValue.of("string"));
|
||||
record.addValue(LtValue.of(1));
|
||||
assertArrayEquals(new byte[]{9, 1, 3, 25, 9, 115, 116, 114, 105, 110, 103}, record.toBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testStringIntFloatValues() {
|
||||
LtRecord record = new LtRecord(1);
|
||||
record.addValue(LtValue.of("string"));
|
||||
record.addValue(LtValue.of(2));
|
||||
assertArrayEquals(new byte[]{10, 1, 3, 25, 1, 115, 116, 114, 105, 110, 103, 2}, record.toBytes());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
package nl.sanderhautvast.sqlighter;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.data.LtRecord;
|
||||
import nl.sanderhautvast.sqlighter.data.LtValue;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.sql.*;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/*
|
||||
* end-to-end test that creates an SQLite database and tries to read it
|
||||
*/
|
||||
public class SchemaCreationTests {
|
||||
|
||||
@Test
|
||||
public void newDatabase() throws SQLException, IOException {
|
||||
DatabaseBuilder databaseBuilder = new DatabaseBuilder();
|
||||
databaseBuilder.addSchema("foo",
|
||||
"CREATE TABLE foo(bar integer,baz varchar(10), xeno float)");
|
||||
|
||||
|
||||
LtRecord record = new LtRecord(1);
|
||||
record.addValues(LtValue.of(12), LtValue.of("hello1world"), LtValue.of(1.2));
|
||||
databaseBuilder.addRecord(record);
|
||||
LtRecord record2 = new LtRecord(2);
|
||||
record2.addValues(LtValue.of(12), LtValue.of(1.2)); // just 2 values
|
||||
databaseBuilder.addRecord(record2);
|
||||
|
||||
databaseBuilder.build().write("test.db");
|
||||
|
||||
try (Connection connection = DriverManager.getConnection("jdbc:sqlite:test.db");
|
||||
Statement statement = connection.createStatement()) {
|
||||
ResultSet result1 = statement.executeQuery("select rowid, name, tbl_name, rootpage, sql from sqlite_master");
|
||||
assertTrue(result1.next());
|
||||
assertEquals("foo", result1.getString("name"));
|
||||
assertEquals("foo", result1.getString("tbl_name"));
|
||||
assertEquals(2, result1.getInt("rootpage"));
|
||||
assertEquals("CREATE TABLE foo(bar integer,baz varchar(10), xeno float)", result1.getString("sql"));
|
||||
|
||||
// baz is 'defined' as varchar,
|
||||
// but it's translated to colummn #2
|
||||
// and that contains 1.2 as double! (in row 2)
|
||||
// and the query runs without errors
|
||||
ResultSet result3 = statement.executeQuery("select baz from foo where ROWID = 2");
|
||||
assertTrue(result3.next());
|
||||
assertEquals(1.2, result3.getDouble(1));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test100Records() throws SQLException, IOException {
|
||||
DatabaseBuilder builder = new DatabaseBuilder();
|
||||
builder.addSchema("foo", "CREATE TABLE foo(bar integer,baz varchar(2000))");
|
||||
|
||||
for (int i = 0; i < 100; i++) {
|
||||
LtRecord record = new LtRecord(i);
|
||||
record.addValues(LtValue.of(12), LtValue.of("helloworld".repeat(200)));
|
||||
builder.addRecord(record);
|
||||
}
|
||||
|
||||
builder.build().write("test.db");
|
||||
|
||||
try (Connection connection = DriverManager.getConnection("jdbc:sqlite:test.db");
|
||||
Statement statement = connection.createStatement()) {
|
||||
ResultSet result1 = statement.executeQuery("select rowid, name, tbl_name, rootpage, sql from sqlite_master");
|
||||
assertTrue(result1.next());
|
||||
assertEquals("foo", result1.getString("name"));
|
||||
assertEquals("foo", result1.getString("tbl_name"));
|
||||
assertEquals(2, result1.getInt("rootpage"));
|
||||
assertEquals("CREATE TABLE foo(bar integer,baz varchar(2000))", result1.getString("sql"));
|
||||
|
||||
|
||||
ResultSet result2 = statement.executeQuery("select count(*) from foo");
|
||||
assertTrue(result2.next());
|
||||
assertEquals(100, result2.getInt(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
package nl.sanderhautvast.sqlighter;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.data.LtValue;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
|
||||
class SerialTypesTests {
|
||||
|
||||
@Test
|
||||
void testInteger0() {
|
||||
ByteBuffer out = ByteBuffer.allocate(1);
|
||||
LtValue.of(0).writeType(out);
|
||||
assertArrayEquals(new byte[]{8}, out.array());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testInteger1() {
|
||||
ByteBuffer out = ByteBuffer.allocate(1);
|
||||
LtValue.of(1).writeType(out);
|
||||
assertArrayEquals(new byte[]{9}, out.array());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testInteger2() {
|
||||
ByteBuffer out = ByteBuffer.allocate(1);
|
||||
LtValue.of(2).writeType(out);
|
||||
assertArrayEquals(new byte[]{1}, out.array());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testInteger128() {
|
||||
ByteBuffer out = ByteBuffer.allocate(1);
|
||||
LtValue.of(128).writeType(out);
|
||||
assertArrayEquals(new byte[]{2}, out.array());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testString() {
|
||||
// 'helloworld' is length 10 in characters
|
||||
// 10*2+13 = 33 = 0x21
|
||||
ByteBuffer out = ByteBuffer.allocate(1);
|
||||
LtValue.of("helloworld").writeType(out);
|
||||
assertArrayEquals(new byte[]{0x21}, out.array());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBlob() {
|
||||
// 'helloworld' is length 10 in bytes
|
||||
// 10*2+12 = 32 = 0x20
|
||||
ByteBuffer out = ByteBuffer.allocate(1);
|
||||
LtValue.of("helloworld".getBytes(StandardCharsets.UTF_8)).writeType(out);
|
||||
assertArrayEquals(new byte[]{0x20}, out.array());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFloat() {
|
||||
ByteBuffer out = ByteBuffer.allocate(1);
|
||||
LtValue.of(1.0).writeType(out);
|
||||
assertArrayEquals(new byte[]{7}, out.array());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
package nl.sanderhautvast.sqlighter;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.data.LtValue;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
|
||||
public class ToBytesTests {
|
||||
|
||||
@Test
|
||||
public void testInteger0() {
|
||||
// 0 and 1 are special values that are encoded in the type itself, which saves 8 bits
|
||||
ByteBuffer out = ByteBuffer.allocate(0);
|
||||
LtValue.of(0).writeValue(out);
|
||||
assertArrayEquals(new byte[0], out.array());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInteger1() {
|
||||
// 0 and 1 are special values that are encoded in the type itself, which saves 8 bits
|
||||
ByteBuffer out = ByteBuffer.allocate(0);
|
||||
LtValue.of(1).writeValue(out);
|
||||
assertArrayEquals(new byte[]{}, out.array());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInteger2() {
|
||||
ByteBuffer out = ByteBuffer.allocate(1);
|
||||
LtValue.of(2).writeValue(out);
|
||||
assertArrayEquals(new byte[]{0x02}, out.array());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInteger128() {
|
||||
ByteBuffer out = ByteBuffer.allocate(2);
|
||||
LtValue.of(128).writeValue(out);
|
||||
assertArrayEquals(new byte[]{0,-128}, out.array()); //0x80 as signed byte
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testString() {
|
||||
// 'helloworld' is length 10 in characters
|
||||
// 10*2+13 = 33 = 0x21
|
||||
ByteBuffer out = ByteBuffer.allocate(10);
|
||||
LtValue.of("helloworld").writeValue(out);
|
||||
assertArrayEquals(new byte[]{0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64}, out.array()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBlob() {
|
||||
// 'helloworld' is length 10 in bytes
|
||||
// 10*2+12 = 32 = 0x20
|
||||
ByteBuffer out = ByteBuffer.allocate(10);
|
||||
LtValue.of("helloworld".getBytes(StandardCharsets.UTF_8)).writeValue(out);
|
||||
assertArrayEquals(new byte[]{0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64}, out.array());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFloat() {
|
||||
// tested the expected binary representation against the actual sqlite file
|
||||
ByteBuffer out = ByteBuffer.allocate(8);
|
||||
LtValue.of(1.1).writeValue(out);
|
||||
assertArrayEquals(new byte[]{0x3f, (byte) 0xf1, (byte) 0x99, (byte) 0x99, (byte) 0x99, (byte) 0x99, (byte) 0x99, (byte) 0x9a}, out.array());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
package nl.sanderhautvast.sqlighter;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
public class VarintTests {
|
||||
|
||||
@Test
|
||||
public void testWriteMin1() {
|
||||
assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, -1}, Varint.write(0xffffffffffffffffL));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWrite1() {
|
||||
Varint.write(220);
|
||||
assertArrayEquals(new byte[]{1}, Varint.write(0x01));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWrite() {
|
||||
assertArrayEquals(new byte[]{-1, -1, -1, -1, -1, -1, -1, 127}, Varint.write(0xffffffffffffffL));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testReadMin1() {
|
||||
assertEquals(0xffffffffffffffffL, Varint.read(new byte[]{-1, -1, -1, -1, -1, -1, -1, -1, -1}));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRead1() {
|
||||
assertEquals(1, Varint.read(new byte[]{1}));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRead() {
|
||||
assertEquals(0xffffffffffffffL, Varint.read(new byte[]{-1, -1, -1, -1, -1, -1, -1, 127}));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRead2() {
|
||||
assertEquals(562949953421311L, Varint.read(new byte[]{-1, -1, -1, -1, -1, -1, 127}));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRead3() {
|
||||
assertEquals(4398046511103L, Varint.read(new byte[]{-1, -1, -1, -1, -1, 127}));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRead4() {
|
||||
assertEquals(34359738367L, Varint.read(new byte[]{-1, -1, -1, -1, 127}));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRead5() {
|
||||
assertEquals(268435455, Varint.read(new byte[]{-1, -1, -1, 127}));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRead6() {
|
||||
assertEquals(2097151, Varint.read(new byte[]{-1, -1, 127}));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRead7() {
|
||||
assertEquals( 16383, Varint.read(new byte[]{-1, 127}));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package nl.sanderhautvast.sqlighter.data;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
|
||||
class BlobValueTest {
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
LtValue blobValue = LtValue.of(new byte[]{1, 2, 3, 4, -128});
|
||||
assertArrayEquals(new byte[]{1, 2, 3, 4, -128}, blobValue.getValue());
|
||||
assertArrayEquals(new byte[]{22}, blobValue.getType());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package nl.sanderhautvast.sqlighter.data;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
|
||||
class FloatValueTest {
|
||||
@Test
|
||||
public void test() {
|
||||
LtValue floatValue = LtValue.of(1.5D);
|
||||
ByteBuffer floatValueBytes = ByteBuffer.allocate(8).putDouble(1.5D);
|
||||
assertArrayEquals(
|
||||
floatValueBytes.array(),
|
||||
floatValue.getValue());
|
||||
assertArrayEquals(new byte[]{7}, floatValue.getType());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
package nl.sanderhautvast.sqlighter.data;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
|
||||
class IntegerValueTest {
|
||||
@Test
|
||||
public void testInteger0() {
|
||||
// 0 and 1 are special values that are encoded in the type itself, which saves 8 bits
|
||||
LtValue integerValue = LtValue.of(0);
|
||||
assertArrayEquals(new byte[0], integerValue.getValue());
|
||||
assertArrayEquals(new byte[]{8}, integerValue.getType());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInteger1() {
|
||||
// 0 and 1 are special values that are encoded in the type itself, which saves 8 bits
|
||||
LtValue integerValue = LtValue.of(1);
|
||||
assertArrayEquals(new byte[]{}, integerValue.getValue());
|
||||
assertArrayEquals(new byte[]{9}, integerValue.getType());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInteger2() {
|
||||
LtValue integerValue = LtValue.of(2);
|
||||
assertArrayEquals(new byte[]{0x02}, integerValue.getValue());
|
||||
assertArrayEquals(new byte[]{1}, integerValue.getType());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInteger132() {
|
||||
LtValue integerValue = LtValue.of(132);
|
||||
assertArrayEquals(new byte[]{0,-124}, integerValue.getValue()); //0x80 as signed byte
|
||||
assertArrayEquals(new byte[]{2}, integerValue.getType());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
package nl.sanderhautvast.sqlighter.data;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
|
||||
class StringValueTest {
|
||||
|
||||
@Test
|
||||
public void testString() {
|
||||
String helloworld = "helloworld";
|
||||
LtValue stringValue = LtValue.of(helloworld);
|
||||
assertArrayEquals(new byte[]{0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64}, stringValue.getValue());
|
||||
assertArrayEquals(new byte[]{(byte) (helloworld.length() * 2 + 13)}, stringValue.getType());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
package nl.sanderhautvast.sqlighter.jdbc;
|
||||
|
||||
import nl.sanderhautvast.sqlighter.data.LtRecord;
|
||||
import nl.sanderhautvast.sqlighter.data.LtValue;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.invocation.InvocationOnMock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.stubbing.Answer;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.ResultSetMetaData;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Types;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class ResulSet2SQLiteTest {
|
||||
|
||||
@Mock
|
||||
private ResultSet jdbcResult;
|
||||
|
||||
@Mock
|
||||
private ResultSetMetaData metaData;
|
||||
|
||||
@Test
|
||||
public void testWhatGoesInShouldComeOut() throws SQLException {
|
||||
when(metaData.getColumnCount()).thenReturn(2);
|
||||
when(metaData.getColumnType(1)).thenReturn(Types.INTEGER);
|
||||
when(metaData.getColumnType(2)).thenReturn(Types.VARCHAR);
|
||||
|
||||
when(jdbcResult.getMetaData()).thenReturn(metaData);
|
||||
|
||||
List<List<?>> table = List.of(
|
||||
List.of(1L, "Dr. John"),
|
||||
List.of(2L, "Allen Toussaint"));
|
||||
StatefulAnswer inputRows = new StatefulAnswer(table.iterator());
|
||||
|
||||
when(jdbcResult.next()).thenAnswer(invocationOnMock -> inputRows.hasNext());
|
||||
when(jdbcResult.getLong(anyInt())).thenAnswer(inputRows);
|
||||
when(jdbcResult.getString(anyInt())).thenAnswer(inputRows);
|
||||
|
||||
ArrayList<LtRecord> rows = new ArrayList<>();
|
||||
ResulSet2SQLite resulSet2SQLite = new ResulSet2SQLite(jdbcResult.getMetaData());
|
||||
while (jdbcResult.next()) {
|
||||
rows.add(resulSet2SQLite.mapRow(jdbcResult));
|
||||
}
|
||||
assertEquals(2, rows.size());
|
||||
LtRecord record = rows.get(0);
|
||||
LtValue value = record.getValue(0);
|
||||
assertArrayEquals(new byte[]{9}, value.getType());
|
||||
|
||||
value = record.getValue(1);
|
||||
assertEquals(8, value.getValue().length);
|
||||
|
||||
record = rows.get(1);
|
||||
value = record.getValue(0);
|
||||
assertArrayEquals(new byte[]{2}, value.getValue());
|
||||
|
||||
value = record.getValue(1);
|
||||
assertEquals(15, value.getValue().length);
|
||||
}
|
||||
|
||||
static class StatefulAnswer implements Answer<Object> {
|
||||
|
||||
private final Iterator<List<?>> records;
|
||||
private List<?> current;
|
||||
|
||||
StatefulAnswer(Iterator<List<?>> records) {
|
||||
this.records = records;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object answer(InvocationOnMock invocationOnMock) {
|
||||
int colIndex = invocationOnMock.getArgument(0);
|
||||
return current.get(colIndex - 1); // ResultSet column index starts with 1
|
||||
}
|
||||
|
||||
boolean hasNext() {
|
||||
boolean hasNext = records.hasNext();
|
||||
if (hasNext) {
|
||||
current = records.next();
|
||||
}
|
||||
return hasNext;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue