From 0ee6baa77665501902fde0c0595659663fded489 Mon Sep 17 00:00:00 2001 From: Sander Hautvast Date: Tue, 8 Nov 2022 08:04:21 +0100 Subject: [PATCH] replaced json rest server with offline sqlite --- src/main/java/perfix/Agent.java | 45 ++- .../perfix/instrument/ClassInstrumentor.java | 27 +- src/main/java/perfix/server/HTTPServer.java | 114 ------ .../java/perfix/server/PerfixController.java | 25 -- .../perfix/server/json/JSONSerializer.java | 21 -- .../java/perfix/server/json/Serializer.java | 45 --- .../json/SerializerCreationException.java | 10 - .../perfix/server/json/SerializerFactory.java | 5 - .../server/json/SynthSerializerFactory.java | 334 ------------------ src/main/java/sqlighter/Database.java | 216 +++++++++++ src/main/java/sqlighter/DatabaseBuilder.java | 76 ++++ src/main/java/sqlighter/SQLiteConstants.java | 43 +++ src/main/java/sqlighter/SchemaRecord.java | 59 ++++ src/main/java/sqlighter/Varint.java | 169 +++++++++ src/main/java/sqlighter/data/Record.java | 102 ++++++ src/main/java/sqlighter/data/Value.java | 123 +++++++ src/main/java/sqlighter/page/Page.java | 156 ++++++++ src/main/java/sqlighter/page/PageCache.java | 38 ++ .../java/sqlighter/page/PageCacheFactory.java | 20 ++ src/main/java/sqlighter/page/PageType.java | 9 + src/main/resources/favicon.ico | 13 - 21 files changed, 1058 insertions(+), 592 deletions(-) delete mode 100644 src/main/java/perfix/server/HTTPServer.java delete mode 100644 src/main/java/perfix/server/PerfixController.java delete mode 100644 src/main/java/perfix/server/json/JSONSerializer.java delete mode 100644 src/main/java/perfix/server/json/Serializer.java delete mode 100644 src/main/java/perfix/server/json/SerializerCreationException.java delete mode 100644 src/main/java/perfix/server/json/SerializerFactory.java delete mode 100644 src/main/java/perfix/server/json/SynthSerializerFactory.java create mode 100644 src/main/java/sqlighter/Database.java create mode 100644 src/main/java/sqlighter/DatabaseBuilder.java create mode 100644 src/main/java/sqlighter/SQLiteConstants.java create mode 100644 src/main/java/sqlighter/SchemaRecord.java create mode 100644 src/main/java/sqlighter/Varint.java create mode 100644 src/main/java/sqlighter/data/Record.java create mode 100644 src/main/java/sqlighter/data/Value.java create mode 100644 src/main/java/sqlighter/page/Page.java create mode 100644 src/main/java/sqlighter/page/PageCache.java create mode 100644 src/main/java/sqlighter/page/PageCacheFactory.java create mode 100644 src/main/java/sqlighter/page/PageType.java delete mode 100644 src/main/resources/favicon.ico diff --git a/src/main/java/perfix/Agent.java b/src/main/java/perfix/Agent.java index 55a20ca..f5defa7 100644 --- a/src/main/java/perfix/Agent.java +++ b/src/main/java/perfix/Agent.java @@ -1,42 +1,63 @@ package perfix; +import sqlighter.data.Value; import perfix.instrument.Instrumentor; -import perfix.server.HTTPServer; +import sqlighter.DatabaseBuilder; +import sqlighter.data.Record; +import java.io.IOException; import java.lang.instrument.Instrumentation; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.LongAdder; import static java.util.Arrays.asList; -import static java.util.Arrays.stream; public class Agent { - private static final String PORT_PROPERTY = "perfix.port"; private static final String INCLUDES_PROPERTY = "perfix.includes"; - private static final String DEFAULT_PORT = "2048"; private static final String MESSAGE = " --- Perfix agent active"; + private static final DatabaseBuilder databaseBuilder = new DatabaseBuilder(); public static void premain(String agentArgs, Instrumentation instrumentation) { System.out.println(MESSAGE); - - int port = Integer.parseInt(System.getProperty(PORT_PROPERTY, DEFAULT_PORT)); - Instrumentor.create(determineIncludes()).instrumentCode(instrumentation); + System.out.println("Instrumenting " + System.getProperty(INCLUDES_PROPERTY)); - new HTTPServer(port).start(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + LongAdder rowid = new LongAdder(); + try { + Registry.sortedMethodsByDuration().values() + .forEach(r -> { + rowid.increment(); + Record record = new Record(rowid.intValue()); + record.addValues(Value.of(r.getName()), Value.of(r.getInvocations()), Value.of(r.getAverage() / 1_000_000F), Value.of(r.getTotalDuration() / 1_000_000F)); + databaseBuilder.addRecord(record); + }); + databaseBuilder.addSchema("results", "create table results(name varchar(100), invocations integer, average float, total float)"); + databaseBuilder.build().write(Files.newByteChannel(Paths.get("results.sqlite"), StandardOpenOption.WRITE, StandardOpenOption.CREATE)); + } catch (IOException e) { + throw new RuntimeException(e); + } + })); } private static List determineIncludes() { - String includesPropertyValue = System.getProperty(INCLUDES_PROPERTY); - if (includesPropertyValue==null){ + String includesPropertyValue = System.getProperty(INCLUDES_PROPERTY).replaceAll("\\.", "/"); + if (includesPropertyValue == null) { System.out.println("WARNING: perfix.includes not set "); return Collections.emptyList(); - } - return new ArrayList<>(asList(includesPropertyValue.split(","))); + } else { + ArrayList includes = new ArrayList<>(asList(includesPropertyValue.split(","))); + System.out.println("includes classes: " + includes + "*"); + return includes; + } } } diff --git a/src/main/java/perfix/instrument/ClassInstrumentor.java b/src/main/java/perfix/instrument/ClassInstrumentor.java index 2bf9407..a9e4878 100644 --- a/src/main/java/perfix/instrument/ClassInstrumentor.java +++ b/src/main/java/perfix/instrument/ClassInstrumentor.java @@ -50,20 +50,21 @@ public class ClassInstrumentor extends Instrumentor { return servletInstrumentor.instrumentServlet(ctClass, uninstrumentedByteCode); } - if (jdbcInstrumentor.isJdbcStatementImpl(resource, ctClass)) { - return jdbcInstrumentor.instrumentJdbcStatement(ctClass, uninstrumentedByteCode); - } +// if (jdbcInstrumentor.isJdbcStatementImpl(resource, ctClass)) { +// return jdbcInstrumentor.instrumentJdbcStatement(ctClass, uninstrumentedByteCode); +// } +// +// if (jdbcInstrumentor.isJdbcConnectionImpl(resource, ctClass)) { +// return jdbcInstrumentor.instrumentJdbcConnection(ctClass, uninstrumentedByteCode); +// } - if (jdbcInstrumentor.isJdbcConnectionImpl(resource, ctClass)) { - return jdbcInstrumentor.instrumentJdbcConnection(ctClass, uninstrumentedByteCode); - } - - if (jdbcInstrumentor.isJdbcPreparedStatement(resource)) { - return jdbcInstrumentor.instrumentJdbcPreparedStatement(ctClass, uninstrumentedByteCode); - } - if (jdbcInstrumentor.isJdbcPreparedStatementImpl(resource, ctClass)) { - return jdbcInstrumentor.instrumentJdbcPreparedStatementImpl(ctClass, uninstrumentedByteCode); - } +// if (jdbcInstrumentor.isJdbcPreparedStatement(resource)) { +// return jdbcInstrumentor.instrumentJdbcPreparedStatement(ctClass, uninstrumentedByteCode); +// } +// if (jdbcInstrumentor.isJdbcPreparedStatementImpl(resource, ctClass)) { +// return jdbcInstrumentor.instrumentJdbcPreparedStatementImpl(ctClass, uninstrumentedByteCode); +// } +// System.out.println(resource); if (shouldInclude(resource, includes)) { return instrumentMethods(ctClass, uninstrumentedByteCode); } diff --git a/src/main/java/perfix/server/HTTPServer.java b/src/main/java/perfix/server/HTTPServer.java deleted file mode 100644 index aa71131..0000000 --- a/src/main/java/perfix/server/HTTPServer.java +++ /dev/null @@ -1,114 +0,0 @@ -package perfix.server; - -import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpHandler; -import com.sun.net.httpserver.HttpServer; -import perfix.server.json.Serializer; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.function.Function; - -public class HTTPServer implements HttpHandler { - - private static final String DEFAULT_ROUTE = "DEFAULT"; - private final int port; - private final ConcurrentMap> routes = new ConcurrentHashMap<>(); - - public HTTPServer(int port) { - this.port = port; - } - - public void start() { - try { - HttpServer server = HttpServer.create(new InetSocketAddress("localhost", port), 0); - server.createContext("/", this); - server.setExecutor(Executors.newFixedThreadPool(3)); - server.start(); - - PerfixController perfixController = new PerfixController(); - routes.put("/report", perfixController::perfixMetrics); - routes.put("/callstack", perfixController::perfixCallstack); - routes.put("/clear", perfixController::clear); - routes.put(DEFAULT_ROUTE, this::staticContent); - - System.out.println(" --- Perfix http server running. Point your browser to http://localhost:" + port + "/"); - } catch (IOException ioe) { - System.err.println(" --- Couldn't start Perfix http server:\n" + ioe); - } - } - - - @Override - public void handle(HttpExchange exchange) throws IOException { - InputStream response = getResponse(exchange); - - OutputStream outputStream = exchange.getResponseBody(); - exchange.getResponseHeaders().add("Access-Control-Allow-Origin", "*"); - exchange.getResponseHeaders().add("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - int length = response.available(); - exchange.sendResponseHeaders(200, length); - - for (int i = 0; i < length; i++) { - outputStream.write(response.read()); - } - outputStream.flush(); - outputStream.close(); - } - - private InputStream getResponse(HttpExchange exchange) { - String uri = exchange.getRequestURI().toString(); - Object response; - if (routes.get(uri) != null) { - response = routes.get(uri).apply(exchange); - } else { - response = routes.get(DEFAULT_ROUTE).apply(exchange); - } - if (response instanceof InputStream) { - return (InputStream) response; - } else { - exchange.getResponseHeaders().add("Content-Type", "application/json"); - return toStream(Serializer.toJSONString(response)); - } - } - - private InputStream staticContent(HttpExchange exchange) { - String uri = exchange.getRequestURI().toString(); - if (uri.equals("/")) { - uri = "/index.html"; - } - InputStream resource = getClass().getResourceAsStream(uri); - if (resource != null) { - String mimeType; - if (uri.endsWith("css")) { - mimeType = "text/css"; - } else if (uri.endsWith("js")) { - mimeType = "application/ecmascript"; - } else if (uri.equals("/favicon.ico")) { - mimeType = "image/svg+xml"; - } else { - mimeType = "text/html"; - } - exchange.getResponseHeaders().add("Content-Type", mimeType); - return resource; - - } else { - return toStream(notFound()); - } - } - - private String notFound() { - return "NOT FOUND"; - } - - private InputStream toStream(String text) { - return new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)); - } -} diff --git a/src/main/java/perfix/server/PerfixController.java b/src/main/java/perfix/server/PerfixController.java deleted file mode 100644 index 03c0170..0000000 --- a/src/main/java/perfix/server/PerfixController.java +++ /dev/null @@ -1,25 +0,0 @@ -package perfix.server; - -import com.sun.net.httpserver.HttpExchange; -import perfix.MethodNode; -import perfix.Registry; -import perfix.Report; - -import java.util.ArrayList; -import java.util.List; - -public class PerfixController { - - public List perfixMetrics(HttpExchange exchange) { - return new ArrayList<>(Registry.sortedMethodsByDuration().values()); - } - - public List perfixCallstack(HttpExchange exchange) { - return Registry.getCallStack(); - } - - public String clear(HttpExchange exchange) { - Registry.clear(); - return "clear"; - } -} diff --git a/src/main/java/perfix/server/json/JSONSerializer.java b/src/main/java/perfix/server/json/JSONSerializer.java deleted file mode 100644 index 7d11d1a..0000000 --- a/src/main/java/perfix/server/json/JSONSerializer.java +++ /dev/null @@ -1,21 +0,0 @@ -package perfix.server.json; - -import java.util.Formatter; - -public abstract class JSONSerializer { - protected abstract String handle(T object); - - protected Formatter formatter = new Formatter(); - - public String toJSONString(T object) { - if (object == null) { - return ""; - } else if (object instanceof Number || object instanceof Boolean) { - return "" + object.toString(); - } else if (object instanceof CharSequence || object instanceof Character) { - return "\"" + object.toString() + "\""; - } else { - return handle(object); - } - } -} diff --git a/src/main/java/perfix/server/json/Serializer.java b/src/main/java/perfix/server/json/Serializer.java deleted file mode 100644 index fa4f42a..0000000 --- a/src/main/java/perfix/server/json/Serializer.java +++ /dev/null @@ -1,45 +0,0 @@ -package perfix.server.json; - -public class Serializer { - private static SerializerFactory instance = new SynthSerializerFactory(); - - public static String toJSONString(boolean b) { - return Boolean.toString(b); - } - - public static String toJSONString(short s) { - return Short.toString(s); - } - - public static String toJSONString(int i) { - return Integer.toString(i); - } - - public static String toJSONString(float f) { - return Float.toString(f); - } - - public static String toJSONString(double d) { - return Double.toString(d); - } - - public static String toJSONString(long l) { - return Long.toString(l); - } - - public static String toJSONString(char c) { - return "\"" + Character.toString(c) + "\""; - } - - @SuppressWarnings("unchecked") - public static String toJSONString(T o) { - if (o == null) { - return "null"; - } - return instance.createSerializer((Class) o.getClass()).toJSONString(o); - } - - public static void setInstance(SerializerFactory instance) { - Serializer.instance = instance; - } -} diff --git a/src/main/java/perfix/server/json/SerializerCreationException.java b/src/main/java/perfix/server/json/SerializerCreationException.java deleted file mode 100644 index 1743429..0000000 --- a/src/main/java/perfix/server/json/SerializerCreationException.java +++ /dev/null @@ -1,10 +0,0 @@ -package perfix.server.json; - -@SuppressWarnings("serial") -public class SerializerCreationException extends RuntimeException { - - public SerializerCreationException(Throwable t) { - super(t); - } - -} diff --git a/src/main/java/perfix/server/json/SerializerFactory.java b/src/main/java/perfix/server/json/SerializerFactory.java deleted file mode 100644 index 70a293b..0000000 --- a/src/main/java/perfix/server/json/SerializerFactory.java +++ /dev/null @@ -1,5 +0,0 @@ -package perfix.server.json; - -public interface SerializerFactory { - JSONSerializer createSerializer(Class beanjavaClass); -} diff --git a/src/main/java/perfix/server/json/SynthSerializerFactory.java b/src/main/java/perfix/server/json/SynthSerializerFactory.java deleted file mode 100644 index f31de76..0000000 --- a/src/main/java/perfix/server/json/SynthSerializerFactory.java +++ /dev/null @@ -1,334 +0,0 @@ -package perfix.server.json; - -import javassist.*; - -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.logging.Logger; - -import static java.util.Arrays.asList; -import static java.util.Collections.unmodifiableSet; - -public class SynthSerializerFactory implements SerializerFactory { - - private static final Logger log = Logger.getLogger("perfix"); - private static final String STRING = "java.lang.String"; - private static final String BOOLEAN = "java.lang.Boolean"; - private static final String CHARACTER = "java.lang.Character"; - private static final String BYTE = "java.lang.Byte"; - private static final String DOUBLE = "java.lang.Double"; - private static final String FLOAT = "java.lang.Float"; - private static final String LONG = "java.lang.Long"; - private static final String SHORT = "java.lang.Short"; - private static final String INTEGER = "java.lang.Integer"; - - private final static Set wrappersAndString = unmodifiableSet(new HashSet(asList(BOOLEAN, CHARACTER, BYTE, DOUBLE, FLOAT, LONG, SHORT, INTEGER, - STRING))); - - private static final String COLLECTION = "java.util.Collection"; - private static final String LIST = "java.util.List"; - private static final String SET = "java.util.Set"; - private static final List mapInterfaces = Collections.unmodifiableList(asList("java.util.Map", "java.util.concurrent.ConcurrentHashMap")); - - private static final ConcurrentMap> serializers = new ConcurrentHashMap<>(); - private static final String ROOT_PACKAGE = "serializer."; - - private final ClassPool pool = ClassPool.getDefault(); - private CtClass serializerBase; - - SynthSerializerFactory() { - init(); - } - - private static boolean isPrimitiveOrWrapperOrString(CtClass beanClass) { - return beanClass.isPrimitive() || wrappersAndString.contains(beanClass.getName()); - } - - void init() { - try { - serializerBase = pool.get(JSONSerializer.class.getName()); - } catch (NotFoundException e) { - throw new SerializerCreationException(e); - } - } - - - @SuppressWarnings("unchecked") - public JSONSerializer createSerializer(Class beanjavaClass) { - String serializerName = createSerializerName(beanjavaClass); - return (JSONSerializer) serializers.computeIfAbsent(serializerName, key -> { - try { - CtClass beanClass = pool.get(beanjavaClass.getName()); - CtClass serializerClass = pool.makeClass(serializerName, serializerBase); - - addToJsonStringMethod(beanClass, serializerClass); - - return createSerializerInstance(serializerClass); - - } catch (NotFoundException | CannotCompileException | ReflectiveOperationException e) { - log.severe(e.toString()); - throw new SerializerCreationException(e); - } - }); - } - - /* - * create method source, compile it and add it to the class under construction - */ - private void addToJsonStringMethod(CtClass beanClass, CtClass serializerClass) throws - NotFoundException, CannotCompileException { - String body = createToJSONStringMethodSource(beanClass); - serializerClass.addMethod(CtNewMethod.make(body, serializerClass)); - } - - /* - * Creates the source, handling the for JSON different types of classes - */ - private String createToJSONStringMethodSource(CtClass beanClass) throws NotFoundException { - String source = "public String handle(Object object){\n"; - if (beanClass.isArray()) { - source += "\tObject[] array=(Object[])object;\n"; - source += handleArray(beanClass); - } else if (isCollection(beanClass)) { - source += "\tObject[] array=((java.util.Collection)object).toArray();\n"; - source += handleArray(beanClass); - } else if (isMap(beanClass)) { - source += handleMap(beanClass); - } else if (!isPrimitiveOrWrapperOrString(beanClass)) { - List getters = getGetters(beanClass); - if (shouldAddGetterCallers(getters)) { - source = addGetterCallers(beanClass, source, getters); - } - } else { - source += "\treturn \"\";}"; - } - return source; - } - - /* - * Any Collection is converted to an array, after which code is generated to handle the single elements. - * - * A subserializer is created for every single element, but most of the time it will be the same cached instance. - * - * The generated code fills a StringBuilder. The values are generated by the subserializers - */ - private String handleArray(CtClass beanClass) { - String source = "\tjava.util.StringJoiner result=new java.util.StringJoiner(\",\",\"[\",\"]\");\n"; - source += "\tfor (int i=0; i getters) throws - NotFoundException { - int index = 0; - source += "\treturn "; - source += "\"{"; - for (CtMethod getter : getters) { - source = addPair(beanClass, source, getter); - if (index++ < getters.size() - 1) { - source += ","; - } - } - source += "}\";\n}"; - return source; - } - - @SuppressWarnings("unchecked") - private JSONSerializer createSerializerInstance(CtClass serializerClass) throws - CannotCompileException, ReflectiveOperationException { - return (JSONSerializer) pool.toClass(serializerClass).getConstructor().newInstance(); - } - - /* - * custom root package is prepended to avoid the java.lang class in which it's illegal to create new classes - * - * Array marks ( '[]' ) are replaced by the 'Array', Otherwise the SerializerClassName would be syntactically incorrect - */ - public String createSerializerName(Class beanClass) { - return createSerializerName(beanClass.getName()); - } - - public String createSerializerName(String name) { - return ROOT_PACKAGE + name.replaceAll("\\[]", "Array") + "Serializer"; - } - - private boolean isCollection(CtClass beanClass) throws NotFoundException { - List interfaces = new ArrayList<>(asList(beanClass.getInterfaces())); - interfaces.add(beanClass); - return interfaces.stream() - .map(CtClass::getName) - .anyMatch(interfaze -> interfaze.equals(COLLECTION) || interfaze.equals(LIST) || interfaze.equals(SET)); - } - - private boolean isMap(CtClass beanClass) throws NotFoundException { - if (mapInterfaces.contains(beanClass.getName())) { - return true; - } else { - return Arrays.stream(beanClass.getInterfaces()) - .map(CtClass::getName) - .anyMatch(mapInterfaces::contains); - } - } - - /* - * The JSON vernacular for key:value is pair... - */ - private String addPair(CtClass classToSerialize, String source, CtMethod getter) throws NotFoundException { - source += jsonKey(getter); - source += ":"; - source += jsonValue(classToSerialize, getter); - return source; - } - - /* - * derive property key from getter - */ - private String jsonKey(CtMethod getter) { - return "\\\"" + toFieldName(getter.getName()) + "\\\""; - } - - private String jsonValue(CtClass classToSerialize, CtMethod getter) throws NotFoundException { - String source = ""; - CtClass returnType = getter.getReturnType(); - - /* primitives are wrapped so the produced methods adhere to the JSONSerializer interface */ - source = createSubSerializerForReturnTypeAndAddInvocationToSource(classToSerialize, getter, source, returnType); - - return source; - } - - private String createSubSerializerForReturnTypeAndAddInvocationToSource(CtClass classToSerialize, CtMethod - getter, String source, CtClass returnType) { - /* NB there does not seem to be auto(un))boxing nor generic types (or other jdk1.5 stuff) in javassist compileable code */ - - source += "\"+" + Serializer.class.getName() + ".toJSONString("; - - // cast because of lack of generics - source += "(" + cast(regularClassname(classToSerialize.getName())) + "object)." + getter.getName() + "()"; - - source += ")+\""; - return source; - } - - /* - * turns for example 'getValue' into 'value' - */ - private String toFieldName(String name) { - return name.substring(3, 4).toLowerCase() + (name.length() > 4 ? name.substring(4) : ""); - } - - public String regularClassname(String name) { - return name.replaceAll("\\$", "."); - } - - private String cast(String classToSerialize) { - return "(" + classToSerialize + ")"; - } - - /* - * Retrieves getter methods from a class - */ - private List getGetters(CtClass beanClass) { - List methods = new ArrayList(); - List fields = getAllFields(beanClass); - for (CtField field : fields) { - try { - CtMethod method = beanClass.getMethod(getGetterMethod(field), getDescription(field)); - if (Modifier.isPublic(method.getModifiers())) { - methods.add(method); - } - } catch (NotFoundException n) { - // ignore - } - } - return methods; - } - - private String getGetterMethod(CtField field) { - return "get" + field.getName().substring(0, 1).toUpperCase() + field.getName().substring(1); - } - - private List getAllFields(CtClass beanClass) { - try { - List allfields = new ArrayList<>(); - return getAllFields(beanClass, allfields); - } catch (NotFoundException e) { - throw new SerializerCreationException(e); - } - - } - - private List getAllFields(CtClass beanClass, List allfields) throws NotFoundException { - allfields.addAll(asList(beanClass.getDeclaredFields())); - if (beanClass.getSuperclass() != null) { - return getAllFields(beanClass.getSuperclass(), allfields); - } - return allfields; - } - - /* - * is getter list is not empty then callers should be added - */ - boolean shouldAddGetterCallers(List getters) { - return !getters.isEmpty(); - } - - String getDescription(CtField field) throws NotFoundException { - if (field.getType().isArray()) { - return "()[" + innerClassName(field.getType().getName()) + ";"; - } else if (!field.getType().isPrimitive()) { - return "()" + innerClassName(field.getType().getName()) + ";"; - } else { - - return "()" + asPrimitive(field.getType().getName()); - } - } - - String asPrimitive(String name) { - switch (name) { - case "int": - return "I"; - case "byte": - return "B"; - case "float": - return "F"; - case "long": - return "J"; - case "boolean": - return "Z"; - case "char": - return "C"; - case "double": - return "D"; - case "short": - return "S"; - } - return ""; - } - - String innerClassName(String name) { - return "L" + name.replaceAll("\\.", "/").replaceAll("\\[\\]", ""); - } -} diff --git a/src/main/java/sqlighter/Database.java b/src/main/java/sqlighter/Database.java new file mode 100644 index 0000000..2bc3255 --- /dev/null +++ b/src/main/java/sqlighter/Database.java @@ -0,0 +1,216 @@ +package sqlighter; + +import sqlighter.page.Page; +import sqlighter.page.PageCacheFactory; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.ArrayList; +import java.util.List; + +import static sqlighter.SQLiteConstants.*; + +/** + * Limited to one table. As this is the main use case, this will probably not change. + * Please note that you can put whatever in it (does not have to reflect the actual source database structure), + * including for example the result of a complex join + * major #1 TODO find a way to handle sizes that don't fit into memory, like overflow to file or sth + */ +public class Database { + + public static short PAGE_SIZE = 8192; + + private final SchemaRecord schema; + public final int pageSize; + + final List leafPages; + + private int pageCounter = 3; + + /* + * assumes 1 schema record ie 1 table. This might not change + */ + public Database(int pageSize, SchemaRecord schemaRecord, List leafPages) { + this.pageSize = pageSize; + this.schema = schemaRecord; + this.leafPages = leafPages; + } + + public void write(String filename) throws IOException { + try (FileOutputStream outputStream = new FileOutputStream(filename)) { + write(outputStream); + } + } + + public void write(WritableByteChannel channel) throws IOException { + List currentTopLayer = this.leafPages; + int nPages = currentTopLayer.size(); + while (currentTopLayer.size() > 1) { // interior page needed? + currentTopLayer = createInteriorPages(currentTopLayer); + nPages += currentTopLayer.size(); + } + assert !currentTopLayer.isEmpty(); + Page tableRootPage = currentTopLayer.get(0); // + channel.write(createHeaderPage(nPages + 1).getDataBuffer()); + setChildReferencesAndWrite(tableRootPage, channel); + channel.close(); + } + + public void write(OutputStream outputStream) throws IOException { + List currentTopLayer = this.leafPages; + int nPages = currentTopLayer.size(); + while (currentTopLayer.size() > 1) { // interior page needed? + currentTopLayer = createInteriorPages(currentTopLayer); + nPages += currentTopLayer.size(); + } + assert !currentTopLayer.isEmpty(); + Page tableRootPage = currentTopLayer.get(0); // + outputStream.write(createHeaderPage(nPages + 1).getData()); + setChildReferencesAndWrite(tableRootPage, outputStream); + outputStream.close(); + } + + private void setChildReferencesAndWrite(Page page, OutputStream outputStream) { + if (page.isInterior()) { + setChildReferences(page); + } + write(page, outputStream); + PageCacheFactory.getPageCache().release(page); + //recurse + page.getChildren().forEach(child -> setChildReferencesAndWrite(child, outputStream)); + } + + private void setChildReferencesAndWrite(Page page, WritableByteChannel channel) { + if (page.isInterior()) { + setChildReferences(page); + } + write(page, channel); + PageCacheFactory.getPageCache().release(page); + //recurse + page.getChildren().forEach(child -> setChildReferencesAndWrite(child, channel)); + } + + private void write(Page page, OutputStream outputStream) { + try { + outputStream.write(page.getData()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void write(Page page, WritableByteChannel channel) { + try { + channel.write(page.getDataBuffer()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + + private void setChildReferences(Page page) { + page.setForwardPosition(Page.POSITION_CELL_COUNT); + page.putU16(page.getChildren().size() - 1); + + for (int i = 0; i < page.getChildren().size() - 1; i++) { // except right-most pointer + page.setForwardPosition(Page.START + i * 2); + int position = page.getU16(); // read the position that was written in an earlier pass + page.setForwardPosition(position); // go to the cell at that location + page.putU32(pageCounter++); // add page reference + } + page.setForwardPosition(Page.POSITION_RIGHTMOST_POINTER); + page.putU32(pageCounter++); + } + + private Page createHeaderPage(int nPages) { + Page headerPage = Page.newHeader(pageSize); + writeHeader(headerPage, nPages); + int payloadLocationWriteLocation = headerPage.getForwardPosition(); // mark current position + + int payloadLocation = writeSchema(headerPage, schema); //write schema payload from the end + headerPage.setForwardPosition(payloadLocationWriteLocation); // go back to marked position + headerPage.putU16(payloadLocation); //payload start + headerPage.skipForward(1); // the number of fragmented free bytes within the cell content area + headerPage.putU16(payloadLocation); // first cell + return headerPage; + } + + private int writeSchema(Page rootPage, SchemaRecord schemaRecord) { + rootPage.putBackward(schemaRecord.toRecord().toBytes()); + return rootPage.getBackwardPosition(); + } + + private List createInteriorPages(List childPages) { + List interiorPages = new ArrayList<>(); + Page interiorPage = PageCacheFactory.getPageCache().getInteriorPage(); + interiorPage.setKey(childPages.stream().mapToLong(Page::getKey).max().orElse(-1)); + interiorPage.setForwardPosition(Page.START); + int pageIndex; + for (pageIndex = 0; pageIndex < childPages.size() - 1; pageIndex++) { + Page leafPage = childPages.get(pageIndex); + if (interiorPage.getBackwardPosition() < interiorPage.getForwardPosition() + 10) { + interiorPage.setForwardPosition(Page.START_OF_CONTENT_AREA); + interiorPage.putU16(interiorPage.getBackwardPosition()); + interiorPage.skipForward(5); + + interiorPages.add(interiorPage); + + interiorPage = PageCacheFactory.getPageCache().getInteriorPage(); + interiorPage.setForwardPosition(Page.START); + } + addCellWithPageRef(interiorPage, leafPage); + interiorPage.addChild(leafPage); + } + + // write start of payload + interiorPage.setForwardPosition(Page.START_OF_CONTENT_AREA); + interiorPage.putU16(interiorPage.getBackwardPosition()); + interiorPage.skipForward(5); + interiorPage.addChild(childPages.get(pageIndex)); + interiorPages.add(interiorPage); + return interiorPages; + } + + private void addCellWithPageRef(Page interiorPage, Page leafPage) { + byte[] keyAsBytes = Varint.write(leafPage.getKey()); + ByteBuffer cell = ByteBuffer.allocate(6 + keyAsBytes.length); + cell.position(5); + cell.put(keyAsBytes); + + // write cell to page, starting at the end + interiorPage.putBackward(cell.array()); + interiorPage.putU16(interiorPage.getBackwardPosition()); + } + + private void writeHeader(Page rootpage, int nPages) { + rootpage.putU8(MAGIC_HEADER); + rootpage.putU16(rootpage.size()); + rootpage.putU8(FILE_FORMAT_WRITE_VERSION); + rootpage.putU8(FILE_FORMAT_READ_VERSION); + rootpage.putU8(RESERVED_SIZE); + rootpage.putU8(MAX_EMBED_PAYLOAD_FRACTION); + rootpage.putU8(MIN_EMBED_PAYLOAD_FRACTION); + rootpage.putU8(LEAF_PAYLOAD_FRACTION); + rootpage.putU32(FILECHANGE_COUNTER); + rootpage.putU32(nPages);// file size in pages + rootpage.putU32(FREELIST_TRUNK_PAGE_HUMBER);// Page number of the first freelist trunk page. + rootpage.putU32(TOTAL_N_FREELIST_PAGES); + rootpage.putU32(SCHEMA_COOKIE); + rootpage.putU32(SQLITE_SCHEMAVERSION); + rootpage.putU32(SUGGESTED_CACHESIZE); + rootpage.putU32(LARGEST_ROOT_BTREE_PAGE); + rootpage.putU32(ENCODING_UTF8); + rootpage.putU32(USER_VERSION); + rootpage.putU32(VACUUM_MODE_OFF);// True (non-zero) for incremental-vacuum mode. False (zero) otherwise. + rootpage.putU32(APP_ID);// Application ID + rootpage.putU8(FILLER);// Reserved for expansion. Must be zero. + rootpage.putU8(VERSION_VALID_FOR);// The version-valid-for number + rootpage.putU8(SQLITE_VERSION);// SQLITE_VERSION_NUMBER + rootpage.putU8(TABLE_LEAF_PAGE); // leaf table b-tree page for schema + rootpage.putU16(NO_FREE_BLOCKS); // zero if there are no freeblocks + rootpage.putU16(1); // the number of cells on the page + } + +} diff --git a/src/main/java/sqlighter/DatabaseBuilder.java b/src/main/java/sqlighter/DatabaseBuilder.java new file mode 100644 index 0000000..25a5bef --- /dev/null +++ b/src/main/java/sqlighter/DatabaseBuilder.java @@ -0,0 +1,76 @@ +package sqlighter; + +import sqlighter.data.Record; +import sqlighter.page.Page; +import sqlighter.page.PageCacheFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * The database builder is the main interface to create a database. + */ +public class DatabaseBuilder { + + private int pageSize = Database.PAGE_SIZE; + private final List leafPages = new ArrayList<>(); + private Page currentPage; + + private SchemaRecord schemaRecord; + + private int nRecordsOnCurrentPage; + + public DatabaseBuilder() { + createPage(); + } + + public DatabaseBuilder withPageSize(int pageSize) { + this.pageSize = pageSize; + return this; + } + + public void addRecord(final Record record) { + if (currentPageIsFull(record)) { + finishCurrentPage(); + createPage(); + } + currentPage.setKey(record.getRowId()); //gets updated until page is finished + currentPage.putBackward(record.toBytes()); + currentPage.putU16(currentPage.getBackwardPosition()); + nRecordsOnCurrentPage += 1; + } + + public void addSchema(String tableName, String ddl) { + this.schemaRecord = new SchemaRecord(1, tableName, 2, ddl); + } + + public Database build() { + currentPage.setForwardPosition(Page.POSITION_CELL_COUNT); + currentPage.putU16(nRecordsOnCurrentPage); + + if (nRecordsOnCurrentPage > 0) { + currentPage.putU16(currentPage.getBackwardPosition()); + } else { + currentPage.putU16(currentPage.getBackwardPosition() - 1); + } + + return new Database(pageSize, schemaRecord, leafPages); + } + + private boolean currentPageIsFull(Record record) { + return currentPage.getBackwardPosition() - record.getDataLength() < currentPage.getForwardPosition() + 5; + } + + private void finishCurrentPage() { + currentPage.setForwardPosition(Page.POSITION_CELL_COUNT); + currentPage.putU16(nRecordsOnCurrentPage); + currentPage.putU16(currentPage.getBackwardPosition()); + } + + private void createPage() { + currentPage = PageCacheFactory.getPageCache().getLeafPage(); + currentPage.setForwardPosition(8); + leafPages.add(currentPage); + nRecordsOnCurrentPage = 0; + } +} diff --git a/src/main/java/sqlighter/SQLiteConstants.java b/src/main/java/sqlighter/SQLiteConstants.java new file mode 100644 index 0000000..b2b4f0f --- /dev/null +++ b/src/main/java/sqlighter/SQLiteConstants.java @@ -0,0 +1,43 @@ +package sqlighter; + +/** + * special values for SQLite. + * + * See Database File Format + */ +public class SQLiteConstants { + public static final byte[] MAGIC_HEADER = new byte[]{0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00}; + + public static final byte FILE_FORMAT_WRITE_VERSION = 1; // legacy + public static final byte FILE_FORMAT_READ_VERSION = 1; // legacy + public static final byte RESERVED_SIZE = 0; + public static final byte MAX_EMBED_PAYLOAD_FRACTION = 0x40; + public static final byte MIN_EMBED_PAYLOAD_FRACTION = 0x20; + public static final byte LEAF_PAYLOAD_FRACTION = 0x20; + public static final int FILECHANGE_COUNTER = 1; + public static final int FREELIST_TRUNK_PAGE_HUMBER = 0; + public static final int TOTAL_N_FREELIST_PAGES = 0; + public static final int SCHEMA_COOKIE = 1; + public static final int SQLITE_SCHEMAVERSION = 4; + public static final int SUGGESTED_CACHESIZE = 0; + public static final int LARGEST_ROOT_BTREE_PAGE = 0; // zero when not in auto-vacuum mode + + public static final int ENCODING_UTF8 = 1; // The database text encoding. A value of 1 means UTF-8. A value of 2 means UTF-16le. A value of 3 means UTF-16be. + + public static final int USER_VERSION = 0; + public static final int VACUUM_MODE_OFF = 0; // not used + public static final int APP_ID = 0; + public static final byte[] FILLER = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }; // 20 bytes for some future use + public static final byte[] VERSION_VALID_FOR = {0, 0, 0x03, -123}; + public static final byte[] SQLITE_VERSION = {0x00, 0x2e, 0x5F, 0x1A}; + + public static final short NO_FREE_BLOCKS = 0; + + + public static final byte TABLE_LEAF_PAGE = 0x0d; //TODO enum? + public static final byte TABLE_INTERIOR_PAGE = 0x05; + public static final byte INDEX_LEAF_PAGE = 0x0a; + public static final byte INDEX_INTERIOR_PAGE = 0x02; +} \ No newline at end of file diff --git a/src/main/java/sqlighter/SchemaRecord.java b/src/main/java/sqlighter/SchemaRecord.java new file mode 100644 index 0000000..07bd15a --- /dev/null +++ b/src/main/java/sqlighter/SchemaRecord.java @@ -0,0 +1,59 @@ +package sqlighter; + +import sqlighter.data.Record; +import sqlighter.data.Value; + +/* + * Is a record in the sqlites_schema table + * and a special case of a Record + * class is being used for both reading and writing + * + */ +public class SchemaRecord { + + private final long rowid; + private final String tableName; + private long rootpage; + private final String sql; + + public SchemaRecord(long rowid, String tableName, long rootpage, String sql) { + this.rowid = rowid; + this.tableName = tableName; + this.rootpage = rootpage; + this.sql = sql; + } + + public SchemaRecord(long rowid, String tableName, long rootpage) { + this(rowid, tableName, rootpage, null); + } + + public String getTableName() { + return tableName; + } + + public long getRowid() { + return rowid; + } + + public long getRootpage() { + return rootpage; + } + + public String getSql() { + return sql; + } + + public void setRootpage(long rootpage) { + this.rootpage = rootpage; + } + + public Record toRecord(){ + Record record = new Record(rowid); + record.addValue(Value.of("table")); + record.addValue(Value.of(getTableName().toLowerCase())); + record.addValue(Value.of(getTableName().toLowerCase())); + record.addValue(Value.of(getRootpage())); + record.addValue(Value.of(getSql())); + return record; + } +} diff --git a/src/main/java/sqlighter/Varint.java b/src/main/java/sqlighter/Varint.java new file mode 100644 index 0000000..fe8ba5d --- /dev/null +++ b/src/main/java/sqlighter/Varint.java @@ -0,0 +1,169 @@ +package sqlighter; + +import java.nio.ByteBuffer; + +/** + * Writes integers to byte representation like Sqlite's putVarint64 + */ +public final class Varint { + private final static byte[] B0 = new byte[0]; + private final static byte[] B1 = new byte[1]; + private final static byte[] B2 = new byte[2]; + private final static byte[] B3 = new byte[3]; + private final static byte[] B4 = new byte[4]; + private final static byte[] B5 = new byte[5]; + private final static byte[] B6 = new byte[6]; + private final static byte[] B7 = new byte[7]; + private final static byte[] B8 = new byte[8]; + private final static byte[] B9 = new byte[9]; + + private final static byte[][] B = {B0, B1, B2, B3, B4, B5, B6, B7, B8, B9}; + + private Varint() { + } + + public static byte[] write(long v) { + if ((v & ((0xff000000L) << 32)) != 0) { + byte[] result = new byte[9]; + result[8] = (byte) v; + v >>= 8; + for (int i = 7; i >= 0; i--) { + result[i] = (byte) ((v & 0x7f) | 0x80); + v >>= 7; + } + return result; + } else { + int n; + byte[] buf = new byte[8]; + for (n = 0; v != 0; n++, v >>= 7) { + buf[n] = (byte) ((v & 0x7f) | 0x80); + } + buf[0] &= 0x7f; + byte[] result = new byte[n]; + for (int i = 0, j = n - 1; j >= 0; j--, i++) { + result[i] = buf[j]; + } + return result; + } + + + } + + /* + * read a long value from a variable nr of bytes in varint format + * NB the end is encoded in the bytes, and the passed byte array may be bigger, but the + * remainder is not read. It's up to the caller to do it right. + */ + public static long read(byte[] bytes) { + return read(ByteBuffer.wrap(bytes)); + } + + /* + * read a long value from a variable nr of bytes in varint format + * + * copied from the sqlite source, with some java specifics, most notably the addition of + * &0xFF for the right conversion from byte => signed in java, but to be interpreted as unsigned, + * to long + * + * Does not have the issue that the read(byte[] bytes) method has. The nr of bytes read is determined + * by the varint64 format. + * + * TODO write specialized version for u32 + */ + public static long read(ByteBuffer buffer) { + int SLOT_2_0 = 0x001fc07f; + int SLOT_4_2_0 = 0xf01fc07f; + + long a = buffer.get() & 0xFF; + if ((a & 0x80) == 0) { + return a; + } + + long b = buffer.get() & 0xFF; + if ((b & 0x80) == 0) { + a &= 0x7F; + a = a << 7; + a |= b; + return a; + } + + a = a << 14; + a |= (buffer.get() & 0xFF); + if ((a & 0x80) == 0) { + a &= SLOT_2_0; + b &= 0x7F; + b = b << 7; + a |= b; + return a; + } + + a &= SLOT_2_0; + b = b << 14; + b |= (buffer.get() & 0xFF); + if ((b & 0x80) == 0) { + b &= SLOT_2_0; + a = a << 7; + a |= b; + return a; + } + + b &= SLOT_2_0; + long s = a; + a = a << 14; + int m = buffer.get() & 0xFF; + a |= m; + if ((a & 0x80) == 0) { + b = b << 7; + a |= b; + s = s >> 18; + return (s << 32) | a; + } + + s = s << 7; + s |= b; + b = b << 14; + b |= (buffer.get() & 0xFF); + if ((b & 0x80) == 0) { + a &= SLOT_2_0; + a = a << 7; + a |= b; + s = s >> 18; + return (s << 32) | a; + } + + a = a << 14; + a |= (buffer.get() & 0xFF); + if ((a & 0x80) == 0) { + a &= SLOT_4_2_0; + b &= SLOT_2_0; + b = b << 7; + a |= b; + s = s >> 11; + return (s << 32) | a; + } + + a &= SLOT_2_0; + b = b << 14; + b |= (buffer.get() & 0xFF); + if ((b & 0x80) == 0) { + b &= SLOT_4_2_0; + a = a << 7; + a |= b; + s = s >> 4; + return (s << 32) | a; + } + + a = a << 15; + a |= (buffer.get() & 0xFF); + b &= SLOT_2_0; + + b = b << 8; + a |= b; + s = s << 4; + b = m; + b &= 0x7F; + b = b >> 3; + s |= b; + return (s << 32) | a; + } +} diff --git a/src/main/java/sqlighter/data/Record.java b/src/main/java/sqlighter/data/Record.java new file mode 100644 index 0000000..0748a99 --- /dev/null +++ b/src/main/java/sqlighter/data/Record.java @@ -0,0 +1,102 @@ +package sqlighter.data; + +import sqlighter.Varint; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * Record in sqlite database. + * Used for reading and writing. + */ +public final class Record { + + /** + * Suppresses use of the rowIDSequence, to facilitate unittests + */ + public static boolean useDefaultRowId = false; + + // start at 1 + + private final long rowId; + + private final List values = new ArrayList<>(10); + + public Record(long rowId) { + this.rowId = rowId; + } + + public void addValues(Value... values) { + this.values.addAll(Arrays.asList(values)); + } + + public void addValue(Value value) { + this.values.add(value); + } + + /** + * write the record to an array of bytes + */ + public byte[] toBytes() { + int dataLength = getDataLength(); + byte[] lengthBytes = Varint.write(dataLength); + byte[] rowIdBytes = Varint.write(rowId); + + ByteBuffer buffer = ByteBuffer.allocate(lengthBytes.length + rowIdBytes.length + dataLength); + buffer.put(lengthBytes); + buffer.put(rowIdBytes); + + // 'The initial portion of the payload that does not spill to overflow pages.' + int lengthOfEncodedColumnTypes = values.stream().map(Value::getDataType).mapToInt(ar -> ar.length).sum() + 1; + buffer.put(Varint.write(lengthOfEncodedColumnTypes)); + + //types + for (Value value : values) { + value.writeType(buffer); + } + + //values + for (Value value : values) { + value.writeValue(buffer); + } + + return buffer.array(); + } + + public int getDataLength() { + return values.stream().mapToInt(Value::getLength).sum() + 1; + } + + public long getRowId() { + return rowId; + } + + @SuppressWarnings("unused") + public List getValues() { + return values; + } + + /** + * returns the value at the specified column index (0 based) + */ + public Value getValue(int column) { + return values.get(column); + } + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Record record = (Record) o; + return rowId == record.rowId; + } + + @Override + public int hashCode() { + return Objects.hash(rowId); + } +} diff --git a/src/main/java/sqlighter/data/Value.java b/src/main/java/sqlighter/data/Value.java new file mode 100644 index 0000000..a496f71 --- /dev/null +++ b/src/main/java/sqlighter/data/Value.java @@ -0,0 +1,123 @@ +package sqlighter.data; + +import sqlighter.Varint; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +/* + * NB Value classes derive their equality from their identity. I.e. no equals/hashcode + */ +public class Value { + private static final byte FLOAT_TYPE = 7; + + protected final byte[] type; + protected final byte[] value; + protected final int length; + + protected Value(byte[] type, byte[] value) { + this.type = type; + this.value = value; + this.length = type.length + value.length; + } + + /** + * Returns the length of serialType + the length of the value + */ + public int getLength() { + return length; + } + + public void writeType(ByteBuffer buffer) { + buffer.put(type); + } + + public byte[] getDataType() { + return type; + } + + public void writeValue(ByteBuffer buffer) { + buffer.put(value); + } + + public byte[] getValue() { + return value; + } + + + public static Value of(String value) { + return new Value(Varint.write(value == null ? 0 : value.getBytes(StandardCharsets.UTF_8).length * 2L + 13), + value == null ? new byte[0] : value.getBytes(StandardCharsets.UTF_8)); + } + + public static Value of(long value) { + byte[] valueAsBytes = getValueAsBytes(value); + return new Value(getIntegerType(value, valueAsBytes.length), valueAsBytes); + } + + public static Value of(double value) { + return new Value(new byte[]{FLOAT_TYPE}, ByteBuffer.wrap(new byte[8]).putDouble(0, value).array()); + } + + public static Value of(byte[] value) { + return new Value(Varint.write(value.length * 2L + 12), value); + } + + public static byte[] getIntegerType(long value, int bytesLength) { + if (value == 0) { + return new byte[]{8}; + } else if (value == 1) { + return new byte[]{9}; + } else { + if (bytesLength < 5) { + return Varint.write(bytesLength); + } else if (bytesLength < 7) { + return Varint.write(5); + } else return Varint.write(6); + } + } + + /* + * static because it's used in the constructor + */ + public static byte[] getValueAsBytes(long value) { + if (value == 0) { + return new byte[0]; + } else if (value == 1) { + return new byte[0]; + } else { + return longToBytes(value, getLengthOfByteEncoding(value)); + } + } + + public static int getLengthOfByteEncoding(long value) { + long u; + if (value < 0) { + u = ~value; + } else { + u = value; + } + if (u <= 127) { + return 1; + } else if (u <= 32767) { + return 2; + } else if (u <= 8388607) { + return 3; + } else if (u <= 2147483647) { + return 4; + } else if (u <= 140737488355327L) { + return 6; + } else { + return 8; + } + } + + public static byte[] longToBytes(long n, int nbytes) { + byte[] b = new byte[nbytes]; + for (int i = 0; i < nbytes; i++) { + b[i] = (byte) ((n >> (nbytes - i - 1) * 8) & 0xFF); + } + + return b; + } +} diff --git a/src/main/java/sqlighter/page/Page.java b/src/main/java/sqlighter/page/Page.java new file mode 100644 index 0000000..1ede298 --- /dev/null +++ b/src/main/java/sqlighter/page/Page.java @@ -0,0 +1,156 @@ +package sqlighter.page; + +import sqlighter.Database; +import sqlighter.SQLiteConstants; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * Represents a SQLite page + */ +public final class Page { + + public static int POSITION_RIGHTMOST_POINTER = 8; // first position after page header + public static int START = 12; // first position after page header + + public static final int POSITION_CELL_COUNT = 3; + public static final int START_OF_CONTENT_AREA = 5; + + private final byte[] data; + + private final ByteBuffer byteBuffer; + private long key; + + private final List children = new ArrayList<>(); + + private int forwardPosition; + private int backwardPosition; + + private final PageType type; + + static Page newLeaf() { + Page page = new Page(PageType.TABLE_LEAF, Database.PAGE_SIZE); + page.putU8(SQLiteConstants.TABLE_LEAF_PAGE); + page.skipForward(2); + return page; + } + + static Page newInterior() { + Page page = new Page(PageType.TABLE_INTERIOR, Database.PAGE_SIZE); + page.putU8(SQLiteConstants.TABLE_INTERIOR_PAGE); + return page; + } + + public static Page newHeader(int size) { + return new Page(PageType.HEADER, size); + } + + public void addChild(Page child) { + children.add(child); + } + + private Page(PageType type, int size) { + this.type = type; + data = new byte[size]; + this.byteBuffer = ByteBuffer.wrap(data); + forwardPosition = 0; + backwardPosition = size; + } + + public int getForwardPosition() { + return forwardPosition; + } + + public int getBackwardPosition() { + return backwardPosition; + } + + public void setForwardPosition(int forwardPosition) { + this.forwardPosition = forwardPosition; + } + + public void putU16(int value) { + data[forwardPosition] = (byte) ((value >> 8) & 0xFF); + data[forwardPosition + 1] = (byte) (value & 0xFF); + forwardPosition += 2; + } + + public void putU32(long value) { + data[forwardPosition] = (byte) ((value >> 24) & 0xFF); + data[forwardPosition + 1] = (byte) ((value >> 16) & 0xFF); + data[forwardPosition + 2] = (byte) ((value >> 8) & 0xFF); + data[forwardPosition + 3] = (byte) (value & 0xFF); + forwardPosition += 4; + } + + public void putU8(int value) { + data[forwardPosition] = (byte) (value & 0xFF); + forwardPosition += 1; + } + + public void putU8(byte[] value) { + System.arraycopy(value, 0, data, forwardPosition, value.length); + forwardPosition += value.length; + } + + public int getU16() { + return ((data[forwardPosition] & 0xFF) << 8) + (data[forwardPosition + 1] & 0xFF); + } + + public void putBackward(byte[] value) { + backwardPosition -= value.length; + System.arraycopy(value, 0, data, backwardPosition, value.length); + } + + public void setKey(long key) { + this.key = key; + } + + public long getKey() { + return key; + } + + public int size() { + return data.length; + } + + public List getChildren() { + return children; + } + + public byte[] getData() { + return data; + } + + public ByteBuffer getDataBuffer(){ +// byteBuffer.clear(); +// byteBuffer.put(data); // someone mentioned that this single write to the (direct) bytebuffer +// // from a byte array is the fastest way to use it +// byteBuffer.flip(); + return byteBuffer; + } + + public void skipForward(int length) { + this.forwardPosition += length; + } + + public boolean isLeaf() { + return type == PageType.TABLE_LEAF; + } + + public boolean isInterior() { + return type == PageType.TABLE_INTERIOR; + } + + public PageType getType() { + return type; + } + + void reset() { + this.forwardPosition = 0; + this.backwardPosition = Database.PAGE_SIZE; + this.children.clear(); + } +} diff --git a/src/main/java/sqlighter/page/PageCache.java b/src/main/java/sqlighter/page/PageCache.java new file mode 100644 index 0000000..c647be3 --- /dev/null +++ b/src/main/java/sqlighter/page/PageCache.java @@ -0,0 +1,38 @@ +package sqlighter.page; + +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; + +public class PageCache { + + protected final Queue leafPages = new LinkedBlockingQueue<>(); + protected final Queue interiorPages = new LinkedBlockingQueue<>(); + + public Page getInteriorPage() { + Page page = interiorPages.poll(); + if (page == null) { + page = Page.newInterior(); + } else { + page.reset(); + } + return page; + } + + public Page getLeafPage() { + Page page = leafPages.poll(); + if (page == null) { + page = Page.newLeaf(); + } else { + page.reset(); + } + return page; + } + + public void release(Page page) { + if (page.getType() == PageType.TABLE_INTERIOR) { + interiorPages.add(page); + } else if (page.getType() == PageType.TABLE_LEAF) { + leafPages.add(page); + } + } +} diff --git a/src/main/java/sqlighter/page/PageCacheFactory.java b/src/main/java/sqlighter/page/PageCacheFactory.java new file mode 100644 index 0000000..5d23dba --- /dev/null +++ b/src/main/java/sqlighter/page/PageCacheFactory.java @@ -0,0 +1,20 @@ +package sqlighter.page; + +import java.lang.ref.SoftReference; +import java.util.Optional; + +public class PageCacheFactory { + + private static final ThreadLocal> threadlocalPageCache = new ThreadLocal<>(); + + + public static PageCache getPageCache() { + return Optional.ofNullable(threadlocalPageCache.get()) + .map(SoftReference::get) + .orElseGet(() -> { + PageCache pageCache = new PageCache(); + threadlocalPageCache.set(new SoftReference<>(pageCache)); + return pageCache; + }); + } +} diff --git a/src/main/java/sqlighter/page/PageType.java b/src/main/java/sqlighter/page/PageType.java new file mode 100644 index 0000000..847ba3a --- /dev/null +++ b/src/main/java/sqlighter/page/PageType.java @@ -0,0 +1,9 @@ +package sqlighter.page; + +public enum PageType { + TABLE_LEAF, + TABLE_INTERIOR, + INDEX_LEAF, + INDEX_INTERIOR, + HEADER, +} diff --git a/src/main/resources/favicon.ico b/src/main/resources/favicon.ico deleted file mode 100644 index bc684ab..0000000 --- a/src/main/resources/favicon.ico +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - -