From 7b5e61e1759561293626fc009c396bc83e9dae56 Mon Sep 17 00:00:00 2001 From: Shautvast Date: Wed, 24 May 2023 22:09:36 +0200 Subject: [PATCH] initial commit after two days work --- .gitignore | 4 + README.md | 5 + pom.xml | 25 + .../contiguous/ByteHandler.java | 22 + .../contiguous/ContiguousList.java | 500 ++++++++++++++++++ .../contiguous/DoubleHandler.java | 17 + .../contiguous/FloatHandler.java | 14 + .../contiguous/IntegerHandler.java | 28 + .../contiguous/LongHandler.java | 14 + .../contiguous/PropertyHandler.java | 93 ++++ .../contiguous/PropertyHandlerFactory.java | 54 ++ .../contiguous/ShortHandler.java | 19 + .../contiguous/StringHandler.java | 14 + .../contiguous/ValueReader.java | 136 +++++ .../nl/sanderhautvast/contiguous/Varint.java | 160 ++++++ .../sanderhautvast/contiguous/ByteBean.java | 13 + .../contiguous/ContiguousListTest.java | 109 ++++ .../sanderhautvast/contiguous/DoubleBean.java | 16 + .../sanderhautvast/contiguous/FloatBean.java | 16 + .../nl/sanderhautvast/contiguous/IntBean.java | 17 + .../sanderhautvast/contiguous/LongBean.java | 17 + .../sanderhautvast/contiguous/NestedBean.java | 20 + .../sanderhautvast/contiguous/ShortBean.java | 13 + .../sanderhautvast/contiguous/StringBean.java | 20 + 24 files changed, 1346 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 pom.xml create mode 100644 src/main/java/nl/sanderhautvast/contiguous/ByteHandler.java create mode 100644 src/main/java/nl/sanderhautvast/contiguous/ContiguousList.java create mode 100644 src/main/java/nl/sanderhautvast/contiguous/DoubleHandler.java create mode 100644 src/main/java/nl/sanderhautvast/contiguous/FloatHandler.java create mode 100644 src/main/java/nl/sanderhautvast/contiguous/IntegerHandler.java create mode 100644 src/main/java/nl/sanderhautvast/contiguous/LongHandler.java create mode 100644 src/main/java/nl/sanderhautvast/contiguous/PropertyHandler.java create mode 100644 src/main/java/nl/sanderhautvast/contiguous/PropertyHandlerFactory.java create mode 100644 src/main/java/nl/sanderhautvast/contiguous/ShortHandler.java create mode 100644 src/main/java/nl/sanderhautvast/contiguous/StringHandler.java create mode 100644 src/main/java/nl/sanderhautvast/contiguous/ValueReader.java create mode 100644 src/main/java/nl/sanderhautvast/contiguous/Varint.java create mode 100644 src/test/java/nl/sanderhautvast/contiguous/ByteBean.java create mode 100644 src/test/java/nl/sanderhautvast/contiguous/ContiguousListTest.java create mode 100644 src/test/java/nl/sanderhautvast/contiguous/DoubleBean.java create mode 100644 src/test/java/nl/sanderhautvast/contiguous/FloatBean.java create mode 100644 src/test/java/nl/sanderhautvast/contiguous/IntBean.java create mode 100644 src/test/java/nl/sanderhautvast/contiguous/LongBean.java create mode 100644 src/test/java/nl/sanderhautvast/contiguous/NestedBean.java create mode 100644 src/test/java/nl/sanderhautvast/contiguous/ShortBean.java create mode 100644 src/test/java/nl/sanderhautvast/contiguous/StringBean.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4ce1cad --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea/ +*.iml +target/ + diff --git a/README.md b/README.md new file mode 100644 index 0000000..c465064 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +**Design decisions** +* built for speed and efficiency (within java) +* uses SQLite storage with 1 exception: float is stored as f32 +* needs jdk 9 (VarHandles) +* minimal reflection (once per creation of the list) \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..5854c99 --- /dev/null +++ b/pom.xml @@ -0,0 +1,25 @@ + + + 4.0.0 + + nl.sanderhautvast + contiguous + Datastructures with contiguous storage + 1.0-SNAPSHOT + + + 9 + 9 + UTF-8 + + + + org.junit.jupiter + junit-jupiter + 5.8.2 + test + + + \ No newline at end of file diff --git a/src/main/java/nl/sanderhautvast/contiguous/ByteHandler.java b/src/main/java/nl/sanderhautvast/contiguous/ByteHandler.java new file mode 100644 index 0000000..1641f72 --- /dev/null +++ b/src/main/java/nl/sanderhautvast/contiguous/ByteHandler.java @@ -0,0 +1,22 @@ +package nl.sanderhautvast.contiguous; + +import java.lang.invoke.MethodHandle; + +/** + * Stores a byte value. + */ +class ByteHandler extends PropertyHandler { + public ByteHandler(MethodHandle getter, MethodHandle setter) { + super(getter, setter); + } + + @Override + public void store(Byte value, ContiguousList list) { + list.storeByte(value); + } + + @Override + public void setValue(Object instance, Object value) { + super.setValue(instance, ((Long) value).byteValue()); + } +} diff --git a/src/main/java/nl/sanderhautvast/contiguous/ContiguousList.java b/src/main/java/nl/sanderhautvast/contiguous/ContiguousList.java new file mode 100644 index 0000000..44b0532 --- /dev/null +++ b/src/main/java/nl/sanderhautvast/contiguous/ContiguousList.java @@ -0,0 +1,500 @@ +package nl.sanderhautvast.contiguous; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.function.UnaryOperator; + +/** + * Experimental List implementation + * Behaves like an ArrayList in that it's resizable and indexed. + * The difference is that it uses an efficiently dehydrated version of the object in a cpu cache friendly, contiguous storage in a bytearray, + * without object instance overhead. + *

+ * Only uses reflection api on creation of the list. + * Adding/Retrieving/Deleting depend on VarHandles and are aimed to be O(L) runtime complexity + * where L is the nr of attributes to get/set from the objects (recursively).So O(1) for length of the list + *

+ * The experiment is to see if performance gains from the memory layout make up for this added overhead + *

+ * Employs the SQLite style of data storage, most notably integer numbers are stored with variable byte length + *

+ * The classes stored in DehydrateList MUST have a no-args constructor. + *

+ * Like ArrayList mutating operations are not synchronized. + *

+ * Does not allow null elements. + *

+ * Implements java.util.List but some methods are not (yet) implemented mainly because they don't make much sense + * performance-wise, like the indexed add and set methods. They mess with the memory layout. The list is meant to + * be appended at the tail. + */ +public class ContiguousList implements List { + + private static final byte[] DOUBLE_TYPE = {7}; + private static final byte[] FLOAT_TYPE = {10}; // not in line with SQLite anymore + private static final int STRING_OFFSET = 13; + private static final int BYTES_OFFSET = 12; // blob TODO decide if include + public static final int MAX_24BITS = 8388607; + public static final long MAX_48BITS = 140737488355327L; + + /* + * storage for dehydated objects + */ + private ByteBuffer data = ByteBuffer.allocate(32); + + private int currentValueIndex; + + private int[] valueIndices = new int[10]; + + private int size; + + private final Class type; + + private final List propertyHandlers = new LinkedList<>(); + + public ContiguousList(Class type) { + this.type = type; + //have to make this recursive + inspectType(type, new ArrayList<>()); + valueIndices[0] = 0; + } + + /* + * Get a list of setters and getters to execute later on to get/set the values of the object + * + * The order of excution is crucial, ie MUST be the same as the order in the stored data. + * + * The advantage of the current implementation is that the binary data is not aware of the actual + * object graph. It only knows the 'primitive' values. + */ + private void inspectType(Class type, List childGetters) { + try { + final MethodHandles.Lookup lookup = MethodHandles.privateLookupIn(type, MethodHandles.lookup()); + Arrays.stream(type.getDeclaredFields()) + .forEach(field -> { + try { + Class fieldType = field.getType(); + MethodHandle getter = lookup.findGetter(type, field.getName(), fieldType); + MethodHandle setter = lookup.findSetter(type, field.getName(), fieldType); + + if (PropertyHandlerFactory.isKnownType(fieldType)) { + PropertyHandler propertyHandler = PropertyHandlerFactory.forType(fieldType, getter, setter); + + // not empty if there has been recursion + if (!childGetters.isEmpty()) { + childGetters.forEach(propertyHandler::addChildGetter); + } + + propertyHandlers.add(propertyHandler); + } else { + // assume nested bean + childGetters.add(getter); + inspectType(fieldType, childGetters); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + public boolean addAll(Collection collection) { + for (E element: collection){ + add(element); + } + return true; + } + + public boolean addAll(int i, Collection collection) { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public boolean removeAll(Collection collection) { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public boolean retainAll(Collection collection) { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public void replaceAll(UnaryOperator operator) { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public void sort(Comparator c) { + throw new RuntimeException("Not implemented"); + } + + public void clear() { + this.currentValueIndex = 0; + this.size = 0; + } + + @Override + public boolean add(E element) { + if (element == null) { + return false; + } + propertyHandlers.forEach(appender -> appender.storeValue(element, this)); + size += 1; + + // keep track of where the objects are stored + if (size > valueIndices.length) { + this.valueIndices = Arrays.copyOf(this.valueIndices, this.valueIndices.length * 2); + } + valueIndices[size] = currentValueIndex; + return true; + } + + @Override + public boolean remove(Object o) { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public boolean containsAll(Collection collection) { + throw new RuntimeException("Not yet implemented"); + } + + @SuppressWarnings("unchecked") + @Override + public E get(int index) { + if (index < 0 || index >= size) { + throw new IndexOutOfBoundsException("index <0 or >" + size); + } + data.position(valueIndices[index]); + try { + E newInstance = (E) type.getDeclaredConstructor().newInstance(); + propertyHandlers.forEach(appender -> { + appender.setValue(newInstance, ValueReader.read(data)); + }); + + return newInstance; + } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | + InvocationTargetException e) { + throw new IllegalStateException(e); + } + } + + @Override + public E set(int i, E e) { + throw new RuntimeException("Not implemented"); + } + + @Override + public void add(int i, E e) { + throw new RuntimeException("Not implemented"); + } + + @Override + public E remove(int i) { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public int indexOf(Object o) { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public int lastIndexOf(Object o) { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public ListIterator listIterator() { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public ListIterator listIterator(int i) { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public List subList(int i, int i1) { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public Spliterator spliterator() { + return List.super.spliterator(); + } + + public int size() { + return size; + } + + @Override + public boolean isEmpty() { + return size == 0; + } + + @Override + public boolean contains(Object o) { + throw new RuntimeException("Not implemented"); + } + + @Override + public Iterator iterator() { + return new Iter(); + } + + @Override + public Object[] toArray() { + Object[] objects = new Object[size]; + for (int i = 0; i < size; i++) { + objects[i] = get(i); + } + return objects; + } + + @Override + public T[] toArray(T[] ts) { + if (size > ts.length) { + return (T[]) toArray(); + } + for (int i = 0; i < size; i++) { + ts[i] = (T) get(i); + } + return ts; + } + + + class Iter implements Iterator { + + private int curIndex = 0; + + @Override + public boolean hasNext() { + return curIndex < size; + } + + @Override + public F next() { + return (F) get(curIndex++); + } + } + + private void store(byte[] bytes) { + ensureCapacity(bytes.length); + data.position(currentValueIndex); // ensures intermittent reads/writes + data.put(bytes); + currentValueIndex += bytes.length; + } + + private void store(byte singlebyte) { + ensureCapacity(1); + data.put(singlebyte); + currentValueIndex += 1; + } + + void storeString(String value) { + if (value == null) { + store(Varint.write(0)); + } else { + byte[] utf = value.getBytes(StandardCharsets.UTF_8); + store(Varint.write(((long) (utf.length) << 1) + STRING_OFFSET)); + store(utf); + } + } + + void storeLong(Long value) { + if (value == null) { + store((byte) 0); + } else { + byte[] valueAsBytes = getValueAsBytes(value); + store(getIntegerType(value, valueAsBytes.length)); + store(valueAsBytes); + } + } + + void storeInteger(Integer value) { + if (value == null) { + store((byte) 0); + } else { + byte[] valueAsBytes = getValueAsBytes(value); + store(getIntegerType(value, valueAsBytes.length)); + store(valueAsBytes); + } + } + + void storeByte(Byte value) { + if (value == null) { + store((byte) 0); + } else { + byte[] valueAsBytes = getValueAsBytes(value); + store(getIntegerType(value, valueAsBytes.length)); + store(valueAsBytes); + } + } + + void storeDouble(Double value) { + if (value == null) { + store((byte) 0); + } else { + store(DOUBLE_TYPE); + store(ByteBuffer.wrap(new byte[8]).putDouble(0, value).array()); + } + } + + void storeFloat(Float value) { + if (value == null) { + store((byte) 0); + } else { + store(FLOAT_TYPE); + store(ByteBuffer.wrap(new byte[4]).putFloat(0, value).array()); + } + } + + byte[] getData() { + return Arrays.copyOfRange(data.array(), 0, currentValueIndex); + } + + int[] getValueIndices() { + return Arrays.copyOfRange(valueIndices, 0, size + 1); + } + + private void ensureCapacity(int length) { + while (currentValueIndex + length > data.capacity()) { + byte[] bytes = this.data.array(); + this.data = ByteBuffer.allocate(this.data.capacity() * 2); + this.data.put(bytes); + } + } + + private static byte[] getValueAsBytes(long value) { + if (value == 0) { + return new byte[0]; + } else if (value == 1) { + return new byte[0]; + } else { + return longToBytes(value, getLengthOfByteEncoding(value)); + } + } + + private static byte[] getValueAsBytes(int value) { + if (value == 0) { + return new byte[0]; + } else if (value == 1) { + return new byte[0]; + } else { + return intToBytes(value, getLengthOfByteEncoding(value)); + } + } + + private static byte[] getValueAsBytes(short value) { + if (value == 0) { + return new byte[0]; + } else if (value == 1) { + return new byte[0]; + } else { + return intToBytes(value, getLengthOfByteEncoding(value)); + } + } + + private static byte[] getValueAsBytes(byte value) { + if (value == 0) { + return new byte[0]; + } else if (value == 1) { + return new byte[0]; + } else { + return new byte[]{value}; + } + } + + private static int getLengthOfByteEncoding(long value) { + long u; + if (value < 0) { + u = ~value; + } else { + u = value; + } + if (u <= Byte.MAX_VALUE) { + return 1; + } else if (u <= Short.MAX_VALUE) { + return 2; + } else if (u <= MAX_24BITS) { + return 3; + } else if (u <= Integer.MAX_VALUE) { + return 4; + } else if (u <= MAX_48BITS) { + return 6; + } else { + return 8; + } + } + + private static int getLengthOfByteEncoding(short value) { + int u; + if (value < 0) { + u = ~value; + } else { + u = value; + } + if (u <= Byte.MAX_VALUE) { + return 1; + } + return 2; + } + + private static int getLengthOfByteEncoding(int value) { + int u; + if (value < 0) { + u = ~value; + } else { + u = value; + } + if (u <= Byte.MAX_VALUE) { + return 1; + } else if (u <= Short.MAX_VALUE) { + return 2; + } else if (u <= MAX_24BITS) { + return 3; + } else { + return 4; + } + } + + private static byte[] intToBytes(int n, int nbytes) { + byte[] b = new byte[nbytes]; + for (int i = 0; i < nbytes; i++) { + b[i] = (byte) ((n >> (nbytes - i - 1) * 8) & 0xFF); + } + + return b; + } + + private static byte[] longToBytes(long n, int nbytes) { + byte[] b = new byte[nbytes]; + for (int i = 0; i < nbytes; i++) { + b[i] = (byte) ((n >> (nbytes - i - 1) * 8) & 0xFF); + } + + return b; + } + + private static byte[] getIntegerType(long value, int bytesLength) { + if (value == 0) { + return new byte[]{8}; + } else if (value == 1) { + return new byte[]{9}; + } else { + if (bytesLength < 5) { + return Varint.write(bytesLength); + } else if (bytesLength < 7) { + return Varint.write(5); + } else return Varint.write(6); + } + } +} diff --git a/src/main/java/nl/sanderhautvast/contiguous/DoubleHandler.java b/src/main/java/nl/sanderhautvast/contiguous/DoubleHandler.java new file mode 100644 index 0000000..9758688 --- /dev/null +++ b/src/main/java/nl/sanderhautvast/contiguous/DoubleHandler.java @@ -0,0 +1,17 @@ +package nl.sanderhautvast.contiguous; + +import java.lang.invoke.MethodHandle; + +/** + * Stores a double value. + */ +class DoubleHandler extends PropertyHandler { + public DoubleHandler(MethodHandle getter, MethodHandle setter) { + super(getter, setter); + } + + @Override + public void store(Double value, ContiguousList list) { + list.storeDouble(value); + } +} diff --git a/src/main/java/nl/sanderhautvast/contiguous/FloatHandler.java b/src/main/java/nl/sanderhautvast/contiguous/FloatHandler.java new file mode 100644 index 0000000..f13a8da --- /dev/null +++ b/src/main/java/nl/sanderhautvast/contiguous/FloatHandler.java @@ -0,0 +1,14 @@ +package nl.sanderhautvast.contiguous; + +import java.lang.invoke.MethodHandle; + +class FloatHandler extends PropertyHandler { + public FloatHandler(MethodHandle getter, MethodHandle setter) { + super(getter, setter); + } + + @Override + public void store(Float value, ContiguousList list) { + list.storeFloat(value); + } +} diff --git a/src/main/java/nl/sanderhautvast/contiguous/IntegerHandler.java b/src/main/java/nl/sanderhautvast/contiguous/IntegerHandler.java new file mode 100644 index 0000000..396319f --- /dev/null +++ b/src/main/java/nl/sanderhautvast/contiguous/IntegerHandler.java @@ -0,0 +1,28 @@ +package nl.sanderhautvast.contiguous; + +import java.lang.invoke.MethodHandle; + +class IntegerHandler extends PropertyHandler { + public IntegerHandler(MethodHandle getter, MethodHandle setter) { + super(getter, setter); + } + + /** + * TODO improve + * it's first extended to long (s64) and then stored with variable length. + * With a little more code for s64, s16 and s8 specifically we can avoid the lenghtening and shortening + */ + @Override + public void store(Integer value, ContiguousList list) { + list.storeInteger((Integer)value); + } + + /* + * Every integer number is considered a (variable length) long in the storage + * This method makes sure it's cast back to the required type (same for byte and short) + */ + @Override + public void setValue(Object instance, Object value) { + super.setValue(instance, ((Long) value).intValue()); + } +} diff --git a/src/main/java/nl/sanderhautvast/contiguous/LongHandler.java b/src/main/java/nl/sanderhautvast/contiguous/LongHandler.java new file mode 100644 index 0000000..25f88ee --- /dev/null +++ b/src/main/java/nl/sanderhautvast/contiguous/LongHandler.java @@ -0,0 +1,14 @@ +package nl.sanderhautvast.contiguous; + +import java.lang.invoke.MethodHandle; + +class LongHandler extends PropertyHandler { + public LongHandler(MethodHandle getter, MethodHandle setter) { + super(getter, setter); + } + + @Override + public void store(Long value, ContiguousList list) { + list.storeLong(value); + } +} diff --git a/src/main/java/nl/sanderhautvast/contiguous/PropertyHandler.java b/src/main/java/nl/sanderhautvast/contiguous/PropertyHandler.java new file mode 100644 index 0000000..557d87f --- /dev/null +++ b/src/main/java/nl/sanderhautvast/contiguous/PropertyHandler.java @@ -0,0 +1,93 @@ +package nl.sanderhautvast.contiguous; + +import java.lang.invoke.MethodHandle; +import java.util.ArrayList; +import java.util.List; + +/* + * Base class for handlers. Its responsibility is to read and write a property from the incoming object to the internal storage. + * + * Can be extended for types that you need to handle. + * + * A property handler is instantiated once per bean property and contains handles to the getter and setter methods + * of the bean that it needs to call 'runtime' (after instantiation of the list), + * ie. when a bean is added or retrieved from the list + */ +public abstract class PropertyHandler { + + private final MethodHandle getter; + private final MethodHandle setter; + + /* + * Apology: + * This was the simplest thing I could think of when trying to accomodate for nested types. + * + * What you end up with after inspection in the DehydrateList is a flat list of getters and setters + * of properties that are in a tree-like structure (primitive properties within (nested) compound types) + * So to read or write a property in a 'root object' (element type in list) with compound property types + * you first have to traverse to the bean graph to the right container (bean) of the property you set/get + * (that is what the childGetters are for) + * + * Ideally you'd do this only once per containing class. In the current implementation it's once per + * property in the containing class. + */ + private final List childGetters = new ArrayList<>(); + + public PropertyHandler(MethodHandle getter, MethodHandle setter) { + this.getter = getter; + this.setter = setter; + } + + /** + * Subclasses call the appropriate store method on the ContiguousList + * + * @param value the value to store + * @param list where to store the value + */ + public abstract void store(T value, ContiguousList list); + + void storeValue(T instance, ContiguousList typedList) { + store(getValue(instance), typedList); + } + + private T getValue(Object instance) { + Object objectToCall = instance; + + try { + for (MethodHandle childGetter:childGetters){ + objectToCall = childGetter.invoke(instance); + } + return (T)getter.invoke(objectToCall); + } catch (Throwable e) { + throw new IllegalStateException(e); + } + } + + /** + * This is used when get() is called on the list. + * As this will create a new instance of the type, it's property values need to be set. + * + * Can be overridden to do transformations on the value after it has been retrieved, but make sure + * to call super.setValue() or the value won't be set. + * + * @param instance the created type + * @param value the value that has been read from ContiguousList storage + */ + public void setValue(Object instance, Object value){ + Object objectToCall = instance; + + try { + for (MethodHandle childGetter:childGetters){ + objectToCall = childGetter.invoke(instance); + } + setter.invokeWithArguments(objectToCall, value); + } catch (Throwable e) { + throw new IllegalStateException(e); + } + + } + + void addChildGetter(MethodHandle childGetter){ + childGetters.add(childGetter); + } +} diff --git a/src/main/java/nl/sanderhautvast/contiguous/PropertyHandlerFactory.java b/src/main/java/nl/sanderhautvast/contiguous/PropertyHandlerFactory.java new file mode 100644 index 0000000..852b514 --- /dev/null +++ b/src/main/java/nl/sanderhautvast/contiguous/PropertyHandlerFactory.java @@ -0,0 +1,54 @@ +package nl.sanderhautvast.contiguous; + +import java.lang.invoke.MethodHandle; +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.Map; + +/* + * Maps the propertyvalue type to a PropertyHandler + */ +final class PropertyHandlerFactory { + private static final Map, Class> APPENDERS = new HashMap<>(); + + private PropertyHandlerFactory() { + } + + static { + APPENDERS.put(String.class, StringHandler.class); + APPENDERS.put(byte.class, ByteHandler.class); + APPENDERS.put(Byte.class, ByteHandler.class); + APPENDERS.put(int.class, IntegerHandler.class); + APPENDERS.put(Integer.class, IntegerHandler.class); + APPENDERS.put(short.class, ShortHandler.class); + APPENDERS.put(Short.class, ShortHandler.class); + APPENDERS.put(long.class, LongHandler.class); + APPENDERS.put(Long.class, LongHandler.class); + APPENDERS.put(float.class, FloatHandler.class); + APPENDERS.put(Float.class, FloatHandler.class); + APPENDERS.put(double.class, DoubleHandler.class); + APPENDERS.put(Double.class, DoubleHandler.class); + //Date/Timestamp + //LocalDate/time + //BigDecimal + //BigInteger + } + + public static boolean isKnownType(Class type) { + return APPENDERS.containsKey(type); + } + + public static PropertyHandler forType(Class type, MethodHandle getter, MethodHandle setter) { + try { + Class appenderClass = APPENDERS.get(type); + if (appenderClass == null) { + throw new IllegalStateException("No ListAppender for " + type.getName()); + } + return appenderClass.getDeclaredConstructor(MethodHandle.class, MethodHandle.class) + .newInstance(getter, setter); + } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | + InvocationTargetException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/src/main/java/nl/sanderhautvast/contiguous/ShortHandler.java b/src/main/java/nl/sanderhautvast/contiguous/ShortHandler.java new file mode 100644 index 0000000..8dd8ad7 --- /dev/null +++ b/src/main/java/nl/sanderhautvast/contiguous/ShortHandler.java @@ -0,0 +1,19 @@ +package nl.sanderhautvast.contiguous; + +import java.lang.invoke.MethodHandle; + +class ShortHandler extends PropertyHandler { + public ShortHandler(MethodHandle getter, MethodHandle setter) { + super(getter, setter); + } + + @Override + public void store(Short value, ContiguousList list) { + list.storeInteger(value == null ? null : value.intValue()); + } + + @Override + public void setValue(Object instance, Object value) { + super.setValue(instance, ((Long) value).shortValue()); + } +} diff --git a/src/main/java/nl/sanderhautvast/contiguous/StringHandler.java b/src/main/java/nl/sanderhautvast/contiguous/StringHandler.java new file mode 100644 index 0000000..28988ac --- /dev/null +++ b/src/main/java/nl/sanderhautvast/contiguous/StringHandler.java @@ -0,0 +1,14 @@ +package nl.sanderhautvast.contiguous; + +import java.lang.invoke.MethodHandle; + +class StringHandler extends PropertyHandler { + public StringHandler(MethodHandle getter, MethodHandle setter) { + super(getter, setter); + } + + @Override + public void store(String value, ContiguousList list) { + list.storeString(value); + } +} diff --git a/src/main/java/nl/sanderhautvast/contiguous/ValueReader.java b/src/main/java/nl/sanderhautvast/contiguous/ValueReader.java new file mode 100644 index 0000000..1a58f33 --- /dev/null +++ b/src/main/java/nl/sanderhautvast/contiguous/ValueReader.java @@ -0,0 +1,136 @@ +package nl.sanderhautvast.contiguous; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/* + * Reads a value from the storage + * the layout is SQLite-like type:Varint, value byte[] + * Varint: byte[] + */ +class ValueReader { + + /** + * Reads a value from the buffer. + * + * @param buffer Bytebuffer containing the storage. + * + * //TODO can we make a typesafe read method? I think so + */ + public static Object read(ByteBuffer buffer) { + long type = Varint.read(buffer); + return read(buffer, type, StandardCharsets.UTF_8); + } + + /** + * Reads a value from the buffer + * + * @param buffer Bytebuffer containing the storage. + * @param columnType type representation borrowed from SQLite + * @param charset database charset + * + * @return the value implementation + */ + private static Object read(ByteBuffer buffer, long columnType, Charset charset) { + if (columnType == 0) { + return null; + } else if (columnType < 6L) { + byte[] integerBytes = new byte[getvalueLengthForType(columnType)]; + buffer.get(integerBytes); + return bytesToLong(integerBytes); + } else if (columnType == 7) { + return buffer.getDouble(); + } else if (columnType == 8) { + return 0; + } else if (columnType == 9) { + return 1; + } else if (columnType == 10) { + return buffer.getFloat(); + } else if (columnType >= 12 && columnType % 2 == 0) { + byte[] bytes = new byte[getvalueLengthForType(columnType)]; + buffer.get(bytes); + return bytes; + } else if (columnType >= 13) { + byte[] bytes = new byte[getvalueLengthForType(columnType)]; + buffer.get(bytes); + return new String(bytes, charset); + } else throw new IllegalStateException("unknown column type" + columnType); + } + + private static int getvalueLengthForType(long columnType) { + // can't switch on long + if (columnType == 0 || columnType == 8 || columnType == 9) { + return 0; + } else if (columnType < 5) { + return (int) columnType; + } else if (columnType == 5) { + return 6; + } else if (columnType == 6 || columnType == 7) { + return 8; + } else if (columnType < 12) { + return -1; + } else { + if (columnType % 2 == 0) { + return (int) ((columnType - 12) >> 1); + } else { + return (int) ((columnType - 13) >> 1); + } + } + } + + static int getLengthOfByteEncoding(long value) { + long u; + if (value < 0) { + u = ~value; + } else { + u = value; + } + if (u <= 127) { + return 1; + } else if (u <= 32767) { + return 2; + } else if (u <= 8388607) { + return 3; + } else if (u <= 2147483647) { + return 4; + } else if (u <= 140737488355327L) { + return 6; + } else { + return 8; + } + } + + public static byte[] getValueAsBytes(long value) { + if (value == 0) { + return new byte[0]; + } else if (value == 1) { + return new byte[0]; + } else { + return longToBytes(value, getLengthOfByteEncoding(value)); + } + } + + public static byte[] longToBytes(long n, int nbytes) { + byte[] b = new byte[nbytes]; + for (int i = 0; i < nbytes; i++) { + b[i] = (byte) ((n >> (nbytes - i - 1) * 8) & 0xFF); + } + + return b; + } + + public static long bytesToLong(final byte[] b) { + long n = 0; + for (int i = 0; i < b.length; i++) { + byte v = b[i]; + int shift = ((b.length - i - 1) * 8); + if (i == 0 && (v & 0x80) != 0) { + n -= (0x80L << shift); + v &= 0x7f; + } + n += ((long)(v&0xFF)) << shift; + } + return n; + } +} diff --git a/src/main/java/nl/sanderhautvast/contiguous/Varint.java b/src/main/java/nl/sanderhautvast/contiguous/Varint.java new file mode 100644 index 0000000..4517cbc --- /dev/null +++ b/src/main/java/nl/sanderhautvast/contiguous/Varint.java @@ -0,0 +1,160 @@ +package nl.sanderhautvast.contiguous; + +import java.nio.ByteBuffer; + +/** + * Writes integers to byte representation like Sqlite's putVarint64 + * not threadsafe (take out B8 and B9 if you need that) + */ +final class Varint { + + //reuse the byte buffers => do not multithread + private static final byte[] B8 = new byte[8]; + private static final byte[] B9 = new byte[9]; + + private Varint() { + } + + public static byte[] write(long v) { + if ((v & ((0xff000000L) << 32)) != 0) { + byte[] result = B9; + result[8] = (byte) v; + v >>= 8; + for (int i = 7; i >= 0; i--) { + result[i] = (byte) ((v & 0x7f) | 0x80); + v >>= 7; + } + return result; + } else { + int n; + byte[] buf = B8; + for (n = 0; v != 0; n++, v >>= 7) { + buf[n] = (byte) ((v & 0x7f) | 0x80); + } + buf[0] &= 0x7f; + byte[] result = new byte[n]; + for (int i = 0, j = n - 1; j >= 0; j--, i++) { + result[i] = buf[j]; + } + return result; + } + } + + /* + * read a long value from a variable nr of bytes in varint format + * NB the end is encoded in the bytes, and the passed byte array may be bigger, but the + * remainder is not read. It's up to the caller to do it right. + */ + public static long read(byte[] bytes) { + return read(ByteBuffer.wrap(bytes)); + } + + /* + * read a long value from a variable nr of bytes in varint format + * + * copied from the sqlite source, with some java specifics, most notably the addition of + * &0xFF for the right conversion from byte => signed in java, but to be interpreted as unsigned, + * to long + * + * Does not have the issue that the read(byte[] bytes) method has. The nr of bytes read is determined + * by the varint64 format. + * + * TODO write specialized version for u32 + */ + public static long read(ByteBuffer buffer) { + int SLOT_2_0 = 0x001fc07f; + int SLOT_4_2_0 = 0xf01fc07f; + + long a = buffer.get() & 0xFF; + if ((a & 0x80) == 0) { + return a; + } + + long b = buffer.get() & 0xFF; + if ((b & 0x80) == 0) { + a &= 0x7F; + a = a << 7; + a |= b; + return a; + } + + a = a << 14; + a |= (buffer.get() & 0xFF); + if ((a & 0x80) == 0) { + a &= SLOT_2_0; + b &= 0x7F; + b = b << 7; + a |= b; + return a; + } + + a &= SLOT_2_0; + b = b << 14; + b |= (buffer.get() & 0xFF); + if ((b & 0x80) == 0) { + b &= SLOT_2_0; + a = a << 7; + a |= b; + return a; + } + + b &= SLOT_2_0; + long s = a; + a = a << 14; + int m = buffer.get() & 0xFF; + a |= m; + if ((a & 0x80) == 0) { + b = b << 7; + a |= b; + s = s >> 18; + return (s << 32) | a; + } + + s = s << 7; + s |= b; + b = b << 14; + b |= (buffer.get() & 0xFF); + if ((b & 0x80) == 0) { + a &= SLOT_2_0; + a = a << 7; + a |= b; + s = s >> 18; + return (s << 32) | a; + } + + a = a << 14; + a |= (buffer.get() & 0xFF); + if ((a & 0x80) == 0) { + a &= SLOT_4_2_0; + b &= SLOT_2_0; + b = b << 7; + a |= b; + s = s >> 11; + return (s << 32) | a; + } + + a &= SLOT_2_0; + b = b << 14; + b |= (buffer.get() & 0xFF); + if ((b & 0x80) == 0) { + b &= SLOT_4_2_0; + a = a << 7; + a |= b; + s = s >> 4; + return (s << 32) | a; + } + + a = a << 15; + a |= (buffer.get() & 0xFF); + b &= SLOT_2_0; + + b = b << 8; + a |= b; + s = s << 4; + b = m; + b &= 0x7F; + b = b >> 3; + s |= b; + return (s << 32) | a; + } +} diff --git a/src/test/java/nl/sanderhautvast/contiguous/ByteBean.java b/src/test/java/nl/sanderhautvast/contiguous/ByteBean.java new file mode 100644 index 0000000..5f4d254 --- /dev/null +++ b/src/test/java/nl/sanderhautvast/contiguous/ByteBean.java @@ -0,0 +1,13 @@ +package nl.sanderhautvast.contiguous; + +class ByteBean { + private byte value; + + ByteBean(byte value) { + this.value = value; + } + + public byte getValue() { + return value; + } +} diff --git a/src/test/java/nl/sanderhautvast/contiguous/ContiguousListTest.java b/src/test/java/nl/sanderhautvast/contiguous/ContiguousListTest.java new file mode 100644 index 0000000..8ed8d07 --- /dev/null +++ b/src/test/java/nl/sanderhautvast/contiguous/ContiguousListTest.java @@ -0,0 +1,109 @@ +package nl.sanderhautvast.contiguous; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class ContiguousListTest { + @Test + public void testString() { + ContiguousList beanList = new ContiguousList<>(StringBean.class); + beanList.add(new StringBean("Douglas Adams")); + + assertArrayEquals(new byte[]{39, 68, 111, 117, 103, 108, 97, 115, 32, 65, 100, 97, 109, 115}, beanList.getData()); + + StringBean douglas = beanList.get(0); + assertEquals("Douglas Adams", douglas.getName()); + + // now add new data to see if existing data remains intact + beanList.add(new StringBean("Ford Prefect")); + + assertEquals("Douglas Adams", beanList.get(0).getName()); + assertEquals("Ford Prefect", beanList.get(1).getName()); + + assertEquals(2, beanList.size()); + } + + @Test + public void testInt() { + ContiguousList beanList = new ContiguousList<>(IntBean.class); + beanList.add(new IntBean(42)); + + assertArrayEquals(new byte[]{1, 42}, + beanList.getData()); + assertEquals(42, beanList.get(0).getValue()); + } + + @Test + public void testLong() { + ContiguousList beanList = new ContiguousList<>(LongBean.class); + beanList.add(new LongBean(42)); + + assertArrayEquals(new byte[]{1, 42}, + beanList.getData()); + assertEquals(42, beanList.get(0).getValue()); + } + + @Test + public void testShort() { + ContiguousList beanList = new ContiguousList<>(ShortBean.class); + beanList.add(new ShortBean((short) 42)); + + assertArrayEquals(new byte[]{1, 42}, + beanList.getData()); + assertArrayEquals(new int[]{0, 2}, beanList.getValueIndices()); + } + + @Test + public void testByte() { + ContiguousList beanList = new ContiguousList<>(ByteBean.class); + beanList.add(new ByteBean((byte) -42)); + + assertArrayEquals(new byte[]{1, -42}, + beanList.getData()); + assertArrayEquals(new int[]{0, 2}, beanList.getValueIndices()); + } + + @Test + public void testNestedBean() { + ContiguousList beanList = new ContiguousList<>(NestedBean.class); + beanList.add(new NestedBean(new StringBean("42"))); + + assertArrayEquals(new byte[]{17, 52, 50}, + beanList.getData()); + assertArrayEquals(new int[]{0, 3}, beanList.getValueIndices()); + } + + @Test + public void testFloat() { + ContiguousList beanList = new ContiguousList<>(FloatBean.class); + beanList.add(new FloatBean(1.1F)); + + + assertEquals(1.1F, beanList.get(0).getValue()); + } + + @Test + public void testDouble() { + ContiguousList beanList = new ContiguousList<>(DoubleBean.class); + beanList.add(new DoubleBean(1.1)); + + assertEquals(1.1, beanList.get(0).getValue()); + } + + @Test + public void testNullFloat() { + ContiguousList beanList = new ContiguousList<>(FloatBean.class); + beanList.add(new FloatBean(null)); + + assertNull(beanList.get(0).getValue()); + } + + @Test + public void testNullString() { + ContiguousList beanList = new ContiguousList<>(StringBean.class); + beanList.add(new StringBean(null)); + + assertNull(beanList.get(0).getName()); + } +} diff --git a/src/test/java/nl/sanderhautvast/contiguous/DoubleBean.java b/src/test/java/nl/sanderhautvast/contiguous/DoubleBean.java new file mode 100644 index 0000000..48e482e --- /dev/null +++ b/src/test/java/nl/sanderhautvast/contiguous/DoubleBean.java @@ -0,0 +1,16 @@ +package nl.sanderhautvast.contiguous; + +class DoubleBean { + private Double value; + + public DoubleBean() { + } + + public DoubleBean(Double value) { + this.value = value; + } + + public Double getValue() { + return value; + } +} diff --git a/src/test/java/nl/sanderhautvast/contiguous/FloatBean.java b/src/test/java/nl/sanderhautvast/contiguous/FloatBean.java new file mode 100644 index 0000000..932447d --- /dev/null +++ b/src/test/java/nl/sanderhautvast/contiguous/FloatBean.java @@ -0,0 +1,16 @@ +package nl.sanderhautvast.contiguous; + +class FloatBean { + private Float value; + + public FloatBean() { + } + + public FloatBean(Float value) { + this.value = value; + } + + public Float getValue() { + return value; + } +} diff --git a/src/test/java/nl/sanderhautvast/contiguous/IntBean.java b/src/test/java/nl/sanderhautvast/contiguous/IntBean.java new file mode 100644 index 0000000..d630103 --- /dev/null +++ b/src/test/java/nl/sanderhautvast/contiguous/IntBean.java @@ -0,0 +1,17 @@ +package nl.sanderhautvast.contiguous; + +class IntBean { + private int value; + + public IntBean(){ + + } + IntBean(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + +} diff --git a/src/test/java/nl/sanderhautvast/contiguous/LongBean.java b/src/test/java/nl/sanderhautvast/contiguous/LongBean.java new file mode 100644 index 0000000..af1d850 --- /dev/null +++ b/src/test/java/nl/sanderhautvast/contiguous/LongBean.java @@ -0,0 +1,17 @@ +package nl.sanderhautvast.contiguous; + +class LongBean { + private long value; + + public LongBean(){ + + } + + LongBean(long value) { + this.value = value; + } + + public long getValue() { + return value; + } +} diff --git a/src/test/java/nl/sanderhautvast/contiguous/NestedBean.java b/src/test/java/nl/sanderhautvast/contiguous/NestedBean.java new file mode 100644 index 0000000..ceda051 --- /dev/null +++ b/src/test/java/nl/sanderhautvast/contiguous/NestedBean.java @@ -0,0 +1,20 @@ +package nl.sanderhautvast.contiguous; + +public class NestedBean { + private StringBean stringBean; + + public NestedBean() { + } + + public NestedBean(StringBean stringBean) { + this.stringBean = stringBean; + } + + public StringBean getStringBean() { + return stringBean; + } + + public void setStringBean(StringBean stringBean) { + this.stringBean = stringBean; + } +} diff --git a/src/test/java/nl/sanderhautvast/contiguous/ShortBean.java b/src/test/java/nl/sanderhautvast/contiguous/ShortBean.java new file mode 100644 index 0000000..cad746a --- /dev/null +++ b/src/test/java/nl/sanderhautvast/contiguous/ShortBean.java @@ -0,0 +1,13 @@ +package nl.sanderhautvast.contiguous; + +class ShortBean { + private short value; + + ShortBean(short value) { + this.value = value; + } + + public short getValue() { + return value; + } +} diff --git a/src/test/java/nl/sanderhautvast/contiguous/StringBean.java b/src/test/java/nl/sanderhautvast/contiguous/StringBean.java new file mode 100644 index 0000000..ff1b0b8 --- /dev/null +++ b/src/test/java/nl/sanderhautvast/contiguous/StringBean.java @@ -0,0 +1,20 @@ +package nl.sanderhautvast.contiguous; + +public class StringBean { + private String name; + + public StringBean(){ + + } + public StringBean(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} \ No newline at end of file