diff --git a/platform/on-air-index/src/com/intellij/platform/onair/storage/api/TransientTree.java b/platform/on-air-index/src/com/intellij/platform/onair/storage/api/TransientTree.java new file mode 100644 index 0000000000000000000000000000000000000000..bb718acbd06ca1c48ccd178e658bc8bf5ddd7165 --- /dev/null +++ b/platform/on-air-index/src/com/intellij/platform/onair/storage/api/TransientTree.java @@ -0,0 +1,29 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.platform.onair.storage.api; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public interface TransientTree { + + int getKeySize(); + + int getBase(); + + @Nullable + byte[] get(@NotNull Novelty.Accessor novelty, @NotNull byte[] key); + + boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull KeyValueConsumer consumer); + + boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull byte[] fromKey, @NotNull KeyValueConsumer consumer); + + TransientTree put(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @NotNull byte[] value); + + TransientTree put(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @NotNull byte[] value, boolean overwrite); + + TransientTree delete(@NotNull Novelty.Accessor novelty, @NotNull byte[] key); + + TransientTree delete(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @Nullable byte[] value); + + TransientTree flush(); +} diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/BTree.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/BTree.java index 300e1d4c0615ed8df442dbbd2e17161011a1582d..a15545d9f8c2825822759b91cac635986cdb3f8b 100644 --- a/platform/on-air-index/src/com/intellij/platform/onair/tree/BTree.java +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/BTree.java @@ -34,6 +34,17 @@ public class BTree implements Tree { Long.MIN_VALUE; } + public BTree(Storage storage, int keySize, Address rootAddress, long startAddress) { + this.storage = storage; + this.keySize = keySize; + this.rootAddress = rootAddress; + this.startAddress = startAddress; + } + + public Storage getStorage() { + return storage; + } + @Override public int getKeySize() { return keySize; @@ -173,15 +184,15 @@ public class BTree implements Tree { @Override public boolean put(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @NotNull byte[] value, boolean overwrite) { final boolean[] result = new boolean[1]; - final BasePage root = loadPage(novelty, rootAddress).getMutableCopy(novelty, this); + final BasePage root = loadPage(novelty, rootAddress).getMutableCopy(novelty); final BasePage newSibling = root.put(novelty, key, value, overwrite, result); if (newSibling != null) { final int metadataOffset = (keySize + BYTES_PER_ADDRESS) * DEFAULT_BASE; final byte[] bytes = new byte[metadataOffset + 2]; bytes[metadataOffset] = INTERNAL; bytes[metadataOffset + 1] = 2; - BasePage.set(0, root.getMinKey(), getKeySize(), bytes, root.address.getLowBytes()); - BasePage.set(1, newSibling.getMinKey(), getKeySize(), bytes, newSibling.address.getLowBytes()); + StoredBTreeUtil.set(0, root.getMinKey(), getKeySize(), bytes, root.address.getLowBytes()); + StoredBTreeUtil.set(1, newSibling.getMinKey(), getKeySize(), bytes, newSibling.address.getLowBytes()); this.rootAddress = new Address(novelty.alloc(bytes)); } else { @@ -198,7 +209,7 @@ public class BTree implements Tree { @Override public boolean delete(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @Nullable byte[] value) { final boolean[] res = new boolean[1]; - rootAddress = delete(novelty, loadPage(novelty, rootAddress).getMutableCopy(novelty, this), key, value, res).address; + rootAddress = StoredBTreeUtil.delete(novelty, loadPage(novelty, rootAddress).getMutableCopy(novelty), key, value, res).address; return res[0]; } @@ -212,6 +223,7 @@ public class BTree implements Tree { return loadPage(novelty, rootAddress).save(novelty, storage, consumer); } + @Override public BTree snapshot() { final Address root = rootAddress; if (root.isNovelty()) { @@ -228,7 +240,7 @@ public class BTree implements Tree { return address.isNovelty() && address.getLowBytes() > startAddress; } - /* package */ BasePage loadPage(@NotNull Novelty.Accessor novelty, Address address) { + public BasePage loadPage(@NotNull Novelty.Accessor novelty, Address address) { final boolean isNovelty = address.isNovelty(); final byte[] bytes = isNovelty ? novelty.lookup(address.getLowBytes()) : storage.lookup(address); if (bytes == null) { @@ -279,21 +291,6 @@ public class BTree implements Tree { return new BTree(storage, keySize, new Address(novelty.alloc(bytes))); } - private static BasePage delete(@NotNull Novelty.Accessor novelty, - @NotNull BasePage root, - @NotNull byte[] key, - @Nullable byte[] value, - boolean[] res) { - if (root.delete(novelty, key, value)) { - root = root.mergeWithChildren(novelty); - res[0] = true; - return root; - } - - res[0] = false; - return root; - } - public interface ToString { String renderKey(byte[] key); diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/BTreeCommon.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/BTreeCommon.java new file mode 100644 index 0000000000000000000000000000000000000000..b072fc2c7704b9fcbc7baa4d9b8756bc00829a5e --- /dev/null +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/BTreeCommon.java @@ -0,0 +1,123 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.platform.onair.tree; + +import com.intellij.platform.onair.storage.api.KeyValueConsumer; +import com.intellij.platform.onair.storage.api.Novelty; +import org.jetbrains.annotations.NotNull; + +import static com.intellij.platform.onair.tree.ByteUtils.compare; + +public class BTreeCommon { + + public static boolean traverseInternalPage(@NotNull final IInternalPage page, + @NotNull Novelty.Accessor novelty, + int fromIndex, + @NotNull byte[] fromKey, + @NotNull KeyValueConsumer consumer) { + boolean first = true; + for (int i = fromIndex; i < page.getSize(); i++) { + IPage child = page.getChild(novelty, i); + if (first) { + if (!child.forEach(novelty, fromKey, consumer)) { + return false; + } + first = false; + } + else { + if (!child.forEach(novelty, consumer)) { + return false; + } + } + } + return true; + } + + @SuppressWarnings("unchecked") + public static <T extends IPage> T insertAt(@NotNull T page, + int base, + @NotNull Novelty.Accessor novelty, + int pos, + byte[] key, + Object child) { + if (!needSplit(page, base)) { + page.insertDirectly(novelty, pos, key, child); + return null; + } + else { + int splitPos = getSplitPos(page, pos); + + final T sibling = (T)page.split(novelty, splitPos, page.getSize() - splitPos); + if (pos >= splitPos) { + // insert into right sibling + page.flush(novelty); + insertAt(sibling, base, novelty, pos - splitPos, key, child); + } + else { + // insert into self + insertAt(sibling, base, novelty, pos, key, child); + } + return sibling; + } + } + + // TODO: extract Policy class + public static boolean needSplit(@NotNull final IPage page, final int base) { + return page.getSize() >= base; + } + + // TODO: extract Policy class + public static int getSplitPos(@NotNull final IPage page, final int insertPosition) { + // if inserting into the most right position - split as 8/1, otherwise - 1/1 + final int pageSize = page.getSize(); + return insertPosition < pageSize ? pageSize >> 1 : (pageSize * 7) >> 3; + } + + // TODO: extract Policy class + public static boolean needMerge(@NotNull final IPage left, @NotNull final IPage right, final int base) { + final int leftSize = left.getSize(); + final int rightSize = right.getSize(); + return leftSize == 0 || rightSize == 0 || leftSize + rightSize <= ((base * 7) >> 3); + } + + public static int binarySearchGuess(byte[] backingArray, int size, int bytesPerKey, int bytesPerAddress, byte[] key) { + int index = binarySearch(backingArray, size, bytesPerKey, bytesPerAddress, key); + if (index < 0) { + index = Math.max(0, -index - 2); + } + return index; + } + + public static int binarySearchRange(byte[] backingArray, int size, int bytesPerKey, int bytesPerAddress, byte[] key) { + int index = binarySearch(backingArray, size, bytesPerKey, bytesPerAddress, key); + if (index < 0) { + index = Math.max(0, -index - 1); + } + return index; + } + + public static int binarySearch(byte[] backingArray, int size, int bytesPerKey, int bytesPerAddress, byte[] key) { + final int bytesPerEntry = bytesPerKey + bytesPerAddress; + + int low = 0; + int high = size - 1; + + while (low <= high) { + final int mid = (low + high) >>> 1; + final int offset = mid * bytesPerEntry; + + final int cmp = compare(backingArray, bytesPerKey, offset, key, bytesPerKey, 0); + if (cmp < 0) { + low = mid + 1; + } + else if (cmp > 0) { + high = mid - 1; + } + else { + // key found + return mid; + } + } + // key not found + return -(low + 1); + } +} diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/BasePage.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/BasePage.java index 7813ed2e057eb2259abf06e3bf986de17fa70952..39cf915e048a8d9fc69d96cda10965ab0e5e4694 100644 --- a/platform/on-air-index/src/com/intellij/platform/onair/tree/BasePage.java +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/BasePage.java @@ -1,7 +1,6 @@ // Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.platform.onair.tree; - import com.intellij.platform.onair.storage.api.*; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -10,11 +9,9 @@ import java.io.PrintStream; import java.util.Arrays; import static com.intellij.platform.onair.tree.BTree.BYTES_PER_ADDRESS; -import static com.intellij.platform.onair.tree.ByteUtils.compare; import static com.intellij.platform.onair.tree.ByteUtils.readUnsignedLong; -import static com.intellij.platform.onair.tree.ByteUtils.writeUnsignedLong; -public abstract class BasePage { +public abstract class BasePage implements IPage { protected final byte[] backingArray; protected final BTree tree; protected final Address address; @@ -28,38 +25,63 @@ public abstract class BasePage { this.size = size; } - @Nullable - protected abstract byte[] get(@NotNull Novelty.Accessor novelty, @NotNull final byte[] key); + public Address getAddress() { + return address; + } + + @Override + public int getSize() { + return size; + } + + @Override + public void flush(@NotNull Novelty.Accessor novelty) { + novelty.update(address.getLowBytes(), backingArray); + } + + @Override + public boolean isTransient() { + return false; + } - protected abstract BasePage getChild(@NotNull Novelty.Accessor novelty, int index); + @Override + public long getMutableAddress() { + if (!address.isNovelty()) { + throw new IllegalStateException("address must be novelty"); + } + return address.getLowBytes(); + } + + @Override + @NotNull + public byte[] getMinKey() { + if (size <= 0) { + throw new ArrayIndexOutOfBoundsException("Page is empty."); + } + + return Arrays.copyOf(backingArray, tree.getKeySize()); // TODO: optimize + } + + @Override + public abstract BasePage mergeWithChildren(@NotNull Novelty.Accessor novelty); @Nullable - protected abstract BasePage put(@NotNull Novelty.Accessor novelty, + public abstract BasePage put(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @NotNull byte[] value, boolean overwrite, boolean[] result); - protected abstract boolean delete(@NotNull Novelty.Accessor novelty, - @NotNull byte[] key, - @Nullable byte[] value); + public abstract boolean delete(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @Nullable byte[] value); - protected abstract BasePage getMutableCopy(@NotNull Novelty.Accessor novelty, BTree tree); + protected abstract BasePage getMutableCopy(@NotNull Novelty.Accessor novelty); - protected abstract BasePage split(@NotNull Novelty.Accessor novelty, int from, int length); - - protected abstract Address save(@NotNull final Novelty.Accessor novelty, @NotNull final Storage storage, @NotNull StorageConsumer consumer); + protected abstract Address save(@NotNull final Novelty.Accessor novelty, + @NotNull final Storage storage, + @NotNull StorageConsumer consumer); protected abstract void dump(@NotNull Novelty.Accessor novelty, @NotNull PrintStream out, int level, BTree.ToString renderer); - protected abstract boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull KeyValueConsumer consumer); - - protected abstract boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull byte[] fromKey, @NotNull KeyValueConsumer consumer); - - protected abstract BasePage mergeWithChildren(@NotNull Novelty.Accessor novelty); - - protected abstract boolean isBottom(); - // WARNING: this method allocates an array protected byte[] getKey(int index) { final int bytesPerKey = tree.getKeySize(); @@ -77,10 +99,6 @@ public abstract class BasePage { return new Address(highBytes, lowBytes); } - protected byte[] getValue(@NotNull Novelty.Accessor novelty, int index) { - return tree.loadLeaf(novelty, getChildAddress(index)); - } - protected void incrementSize() { if (size >= tree.getBase()) { throw new IllegalArgumentException("Can't increase tree page size"); @@ -95,115 +113,6 @@ public abstract class BasePage { setSize(size - value); } - @NotNull - protected byte[] getMinKey() { - if (size <= 0) { - throw new ArrayIndexOutOfBoundsException("Page is empty."); - } - - return Arrays.copyOf(backingArray, tree.getKeySize()); // TODO: optimize - } - - protected int binarySearchGuess(byte[] key) { - int index = binarySearch(key); - if (index < 0) { - index = Math.max(0, -index - 2); - } - return index; - } - - protected int binarySearchRange(byte[] key) { - int index = binarySearch(key); - if (index < 0) { - index = Math.max(0, -index - 1); - } - return index; - } - - protected int binarySearch(byte[] key) { - final int bytesPerKey = tree.getKeySize(); - final int bytesPerEntry = bytesPerKey + BYTES_PER_ADDRESS; - - int low = 0; - int high = size - 1; - - while (low <= high) { - final int mid = (low + high) >>> 1; - final int offset = mid * bytesPerEntry; - - final int cmp = compare(backingArray, bytesPerKey, offset, key, bytesPerKey, 0); - if (cmp < 0) { - low = mid + 1; - } - else if (cmp > 0) { - high = mid - 1; - } - else { - // key found - return mid; - } - } - // key not found - return -(low + 1); - } - - protected void flush(@NotNull Novelty.Accessor novelty) { - novelty.update(address.getLowBytes(), backingArray); - } - - protected void set(int pos, byte[] key, long lowAddressBytes) { - final int bytesPerKey = tree.getKeySize(); - - if (key.length != bytesPerKey) { - throw new IllegalArgumentException("Invalid key length: need " + bytesPerKey + ", got: " + key.length); - } - - set(pos, key, bytesPerKey, backingArray, lowAddressBytes); - } - - protected BasePage insertAt(@NotNull Novelty.Accessor novelty, int pos, byte[] key, long childAddress) { - if (!needSplit(this)) { - insertDirectly(novelty, pos, key, childAddress); - return null; - } - else { - int splitPos = getSplitPos(this, pos); - - final BasePage sibling = split(novelty, splitPos, size - splitPos); - if (pos >= splitPos) { - // insert into right sibling - flush(novelty); - sibling.insertAt(novelty, pos - splitPos, key, childAddress); - } - else { - // insert into self - insertAt(novelty, pos, key, childAddress); - } - return sibling; - } - } - - protected void insertDirectly(@NotNull Novelty.Accessor novelty, final int pos, @NotNull byte[] key, long childAddress) { - if (pos < size) { - copyChildren(pos, pos + 1); - } - set(pos, key, childAddress); - incrementSize(); - flush(novelty); - } - - protected void copyChildren(final int from, final int to) { - if (from >= size) return; - - final int bytesPerEntry = tree.getKeySize() + BYTES_PER_ADDRESS; - - System.arraycopy( - backingArray, from * bytesPerEntry, - backingArray, to * bytesPerEntry, - (size - from) * bytesPerEntry - ); - } - protected void mergeWith(BasePage page) { final int bytesPerEntry = tree.getKeySize() + BYTES_PER_ADDRESS; System.arraycopy(page.backingArray, 0, backingArray, size * bytesPerEntry, page.size); @@ -213,67 +122,9 @@ public abstract class BasePage { backingArray[metadataOffset + 1] = (byte)length; } - private void setSize(int updatedSize) { + protected void setSize(int updatedSize) { final int sizeOffset = ((tree.getKeySize() + BYTES_PER_ADDRESS) * tree.getBase()) + 1; backingArray[sizeOffset] = (byte)updatedSize; this.size = updatedSize; } - - // TODO: extract Policy class - public boolean needSplit(@NotNull final BasePage page) { - return page.size >= tree.getBase(); - } - - // TODO: extract Policy class - public int getSplitPos(@NotNull final BasePage page, final int insertPosition) { - // if inserting into the most right position - split as 8/1, otherwise - 1/1 - final int pageSize = page.size; - return insertPosition < pageSize ? pageSize >> 1 : (pageSize * 7) >> 3; - } - - // TODO: extract Policy class - public boolean needMerge(@NotNull final BasePage left, @NotNull final BasePage right) { - final int leftSize = left.size; - final int rightSize = right.size; - return leftSize == 0 || rightSize == 0 || leftSize + rightSize <= ((tree.getBase() * 7) >> 3); - } - - static void set(int pos, byte[] key, int bytesPerKey, byte[] backingArray, long lowAddressBytes) { - final int offset = (bytesPerKey + BYTES_PER_ADDRESS) * pos; - - // write key - System.arraycopy(key, 0, backingArray, offset, bytesPerKey); - // write address - writeUnsignedLong(lowAddressBytes, 8, backingArray, offset + bytesPerKey); - writeUnsignedLong(0, 8, backingArray, offset + bytesPerKey + 8); - } - - static void set(int pos, byte[] key, int bytesPerKey, byte[] backingArray, byte[] inlineValue) { - int offset = (bytesPerKey + BYTES_PER_ADDRESS) * pos; - - // write key - System.arraycopy(key, 0, backingArray, offset, bytesPerKey); - // write value - offset += bytesPerKey; - System.arraycopy(inlineValue, 0, backingArray, offset, inlineValue.length); - backingArray[offset + BYTES_PER_ADDRESS - 1] = (byte)inlineValue.length; - } - - static void setChild(int pos, int bytesPerKey, byte[] backingArray, long lowAddressBytes, long highAddressBytes) { - final int offset = (bytesPerKey + BYTES_PER_ADDRESS) * pos; - // write address - writeUnsignedLong(lowAddressBytes, 8, backingArray, offset + bytesPerKey); - writeUnsignedLong(highAddressBytes, 8, backingArray, offset + bytesPerKey + 8); - } - - static void setChild(int pos, int bytesPerKey, byte[] backingArray, byte[] inlineValue) { - final int offset = (bytesPerKey + BYTES_PER_ADDRESS) * pos + bytesPerKey; - // write value - System.arraycopy(inlineValue, 0, backingArray, offset, inlineValue.length); - backingArray[offset + BYTES_PER_ADDRESS - 1] = (byte)inlineValue.length; - } - - static void indent(PrintStream out, int level) { - for (int i = 0; i < level; i++) out.print(" "); - } } diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/BottomPage.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/BottomPage.java index 3e282e11a752883e744e3395745f47f0ff354e02..fe376fd221a7987cd6e3fad158f6e0ee8380c965 100644 --- a/platform/on-air-index/src/com/intellij/platform/onair/tree/BottomPage.java +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/BottomPage.java @@ -2,6 +2,7 @@ package com.intellij.platform.onair.tree; import com.intellij.platform.onair.storage.api.*; +import com.intellij.platform.onair.tree.functional.BaseTransientPage; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -9,6 +10,9 @@ import java.io.PrintStream; import java.util.Arrays; import static com.intellij.platform.onair.tree.BTree.BYTES_PER_ADDRESS; +import static com.intellij.platform.onair.tree.StoredBTreeUtil.indent; +import static com.intellij.platform.onair.tree.StoredBTreeUtil.set; +import static com.intellij.platform.onair.tree.StoredBTreeUtil.setChild; public class BottomPage extends BasePage { protected int mask; @@ -20,8 +24,8 @@ public class BottomPage extends BasePage { @Nullable @Override - protected byte[] get(@NotNull Novelty.Accessor novelty, @NotNull byte[] key) { - final int index = binarySearch(key); + public byte[] get(@NotNull Novelty.Accessor novelty, @NotNull byte[] key) { + final int index = BTreeCommon.binarySearch(backingArray, size, tree.getKeySize(), BYTES_PER_ADDRESS, key); if (index >= 0) { return getValue(novelty, index); } @@ -29,7 +33,7 @@ public class BottomPage extends BasePage { } @Override - protected boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull KeyValueConsumer consumer) { + public boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull KeyValueConsumer consumer) { for (int i = 0; i < size; i++) { byte[] key = getKey(i); byte[] value = getValue(novelty, i); @@ -41,8 +45,8 @@ public class BottomPage extends BasePage { } @Override - protected boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull byte[] fromKey, @NotNull KeyValueConsumer consumer) { - for (int i = binarySearchRange(fromKey); i < size; i++) { + public boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull byte[] fromKey, @NotNull KeyValueConsumer consumer) { + for (int i = BTreeCommon.binarySearchRange(backingArray, size, tree.getKeySize(), BYTES_PER_ADDRESS, fromKey); i < size; i++) { byte[] key = getKey(i); byte[] value = getValue(novelty, i); if (!consumer.consume(key, value)) { @@ -54,8 +58,8 @@ public class BottomPage extends BasePage { @Nullable @Override - protected BasePage put(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @NotNull byte[] value, boolean overwrite, boolean[] result) { - int pos = binarySearch(key); + public BasePage put(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @NotNull byte[] value, boolean overwrite, boolean[] result) { + int pos = BTreeCommon.binarySearch(backingArray, size, tree.getKeySize(), BYTES_PER_ADDRESS, key); if (pos >= 0) { if (overwrite) { final int bytesPerEntry = tree.getKeySize() + BYTES_PER_ADDRESS; @@ -66,12 +70,12 @@ public class BottomPage extends BasePage { } else { // key found - if ((mask & (1L << pos)) == 0) { + /*if ((mask & (1L << pos)) == 0) { final Address childAddress = getChildAddress(pos); - /*if (tree.canMutateInPlace(childAddress)) { + if (tree.canMutateInPlace(childAddress)) { novelty.free(childAddress.getLowBytes()); - }*/ - } + } + }*/ final long childAddressLowBytes = novelty.alloc(value); mask &= ~(1 << pos); // drop mask bit @@ -94,7 +98,7 @@ public class BottomPage extends BasePage { page = insertValueAt(novelty, pos, key, value); } else { - page = insertAt(novelty, pos, key, novelty.alloc(value)); + page = BTreeCommon.insertAt(this, tree.getBase(), novelty, pos, key, value); } result[0] = true; tree.incrementSize(); @@ -102,11 +106,11 @@ public class BottomPage extends BasePage { } @Override - protected boolean delete(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @Nullable byte[] value) { - final int pos = binarySearch(key); + public boolean delete(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @Nullable byte[] value) { + final int pos = BTreeCommon.binarySearch(backingArray, size, tree.getKeySize(), BYTES_PER_ADDRESS, key); if (pos < 0) return false; - // tree.addExpiredLoggable(keysAddresses[pos]); + // novelty.free(getChildAddress(pos)); copyChildren(pos + 1, pos); tree.decrementSize(); decrementSize(1); @@ -116,7 +120,7 @@ public class BottomPage extends BasePage { } @Override - protected BottomPage split(@NotNull Novelty.Accessor novelty, int from, int length) { + public BottomPage split(@NotNull Novelty.Accessor novelty, int from, int length) { final BottomPage result = copyOf(novelty, this, from, length); decrementSize(length); flush(novelty); @@ -124,7 +128,7 @@ public class BottomPage extends BasePage { } @Override - protected BottomPage getMutableCopy(@NotNull Novelty.Accessor novelty, BTree tree) { + protected BottomPage getMutableCopy(@NotNull Novelty.Accessor novelty) { if (tree.canMutateInPlace(address)) { return this; } @@ -135,6 +139,11 @@ public class BottomPage extends BasePage { ); } + @Override + public BaseTransientPage getTransientCopy(long epoch) { + throw new UnsupportedOperationException(); // TODO + } + @Override protected Address save(@NotNull Novelty.Accessor novelty, @NotNull Storage storage, @NotNull StorageConsumer consumer) { final byte[] resultBytes = Arrays.copyOf(backingArray, backingArray.length); @@ -159,14 +168,7 @@ public class BottomPage extends BasePage { ByteUtils.writeUnsignedInt(mask ^ 0x80000000, backingArray, bytesPerEntry * tree.getBase() + 2); } - @Override - protected void set(int pos, byte[] key, long lowAddressBytes) { - mask &= ~(1 << pos); // drop mask bit - updateMask(tree.getKeySize() + BYTES_PER_ADDRESS); - super.set(pos, key, lowAddressBytes); - } - - private void setValue(int pos, byte[] key, byte[] value) { + private void setValue(int pos, byte[] key, byte[] value) { mask |= (1 << pos); // set mask bit updateMask(tree.getKeySize() + BYTES_PER_ADDRESS); final int bytesPerKey = tree.getKeySize(); @@ -178,13 +180,13 @@ public class BottomPage extends BasePage { set(pos, key, bytesPerKey, backingArray, value); } - protected BasePage insertValueAt(@NotNull Novelty.Accessor novelty, int pos, byte[] key, byte[] value) { - if (!needSplit(this)) { + private BasePage insertValueAt(@NotNull Novelty.Accessor novelty, int pos, byte[] key, byte[] value) { + if (!BTreeCommon.needSplit(this, tree.getBase())) { insertValueDirectly(novelty, pos, key, value); return null; } else { - int splitPos = getSplitPos(this, pos); + int splitPos = BTreeCommon.getSplitPos(this, pos); final BottomPage sibling = split(novelty, splitPos, size - splitPos); if (pos >= splitPos) { @@ -200,17 +202,6 @@ public class BottomPage extends BasePage { } } - @Override - protected void copyChildren(int from, int to) { - int highBits = mask & (0xFFFFFFFF << from); - int lowBits = mask & ~(0xFFFFFFFF << Math.min(from, to)); - - this.mask = lowBits | highBits << (to - from); - updateMask(tree.getKeySize() + BYTES_PER_ADDRESS); - - super.copyChildren(from, to); - } - private void insertValueDirectly(@NotNull Novelty.Accessor novelty, final int pos, @NotNull byte[] key, @NotNull byte[] value) { if (pos < size) { copyChildren(pos, pos + 1); @@ -220,13 +211,12 @@ public class BottomPage extends BasePage { flush(novelty); } - @Override - protected byte[] getValue(@NotNull Novelty.Accessor novelty, int index) { + private byte[] getValue(@NotNull Novelty.Accessor novelty, int index) { if ((mask & (1L << index)) != 0) { return getInlineValue(index); } else { - return super.getValue(novelty, index); + return tree.loadLeaf(novelty, getChildAddress(index)); } } @@ -243,17 +233,33 @@ public class BottomPage extends BasePage { } @Override - protected BasePage getChild(@NotNull Novelty.Accessor novelty, int index) { - throw new UnsupportedOperationException(); + public BasePage mergeWithChildren(@NotNull Novelty.Accessor novelty) { + return this; } @Override - protected BasePage mergeWithChildren(@NotNull Novelty.Accessor novelty) { - return this; + public void insertDirectly(@NotNull Novelty.Accessor novelty, final int pos, @NotNull byte[] key, Object child) { + if (pos < size) { + copyChildren(pos, pos + 1); + } + + final int bytesPerKey = tree.getKeySize(); + + if (key.length != bytesPerKey) { + throw new IllegalArgumentException("Invalid key length: need " + bytesPerKey + ", got: " + key.length); + } + + mask &= ~(1 << pos); // drop mask bit + updateMask(tree.getKeySize() + BYTES_PER_ADDRESS); + + set(pos, key, bytesPerKey, backingArray, novelty.alloc((byte[])child)); + + incrementSize(); + flush(novelty); } @Override - protected boolean isBottom() { + public boolean isBottom() { return true; } @@ -279,6 +285,24 @@ public class BottomPage extends BasePage { } } + private void copyChildren(int from, int to) { + int highBits = mask & (0xFFFFFFFF << from); + int lowBits = mask & ~(0xFFFFFFFF << Math.min(from, to)); + + this.mask = lowBits | highBits << (to - from); + updateMask(tree.getKeySize() + BYTES_PER_ADDRESS); + + if (from >= size) return; + + final int bytesPerEntry = tree.getKeySize() + BYTES_PER_ADDRESS; + + System.arraycopy( + backingArray, from * bytesPerEntry, + backingArray, to * bytesPerEntry, + (size - from) * bytesPerEntry + ); + } + private static BottomPage copyOf(@NotNull Novelty.Accessor novelty, BottomPage page, int from, int length) { byte[] bytes = new byte[page.backingArray.length]; diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/IInternalPage.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/IInternalPage.java new file mode 100644 index 0000000000000000000000000000000000000000..bcf89815dbb73c7f0f1e8c87238ecd9db55004f9 --- /dev/null +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/IInternalPage.java @@ -0,0 +1,10 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.platform.onair.tree; + +import com.intellij.platform.onair.storage.api.Novelty; +import org.jetbrains.annotations.NotNull; + +public interface IInternalPage extends IPage { + + IPage getChild(@NotNull Novelty.Accessor novelty, final int index); +} diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/IPage.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/IPage.java new file mode 100644 index 0000000000000000000000000000000000000000..1de7806f1171144822344b85867b4d6522d11404 --- /dev/null +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/IPage.java @@ -0,0 +1,45 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.platform.onair.tree; + +import com.intellij.platform.onair.storage.api.KeyValueConsumer; +import com.intellij.platform.onair.storage.api.Novelty; +import com.intellij.platform.onair.tree.functional.BaseTransientPage; +import org.jetbrains.annotations.NotNull; + +public interface IPage { + // meta + + int getSize(); + + boolean isBottom(); + + boolean isTransient(); + + // TODO: cleanup? + + long getMutableAddress(); + + BaseTransientPage getTransientCopy(long epoch); + + // crud + + byte[] getMinKey(); + + byte[] get(Novelty.Accessor novelty, byte[] key); + + boolean forEach(Novelty.Accessor novelty, KeyValueConsumer consumer); + + boolean forEach(Novelty.Accessor novelty, byte[] key, KeyValueConsumer consumer); + + // tree-specific methods + + void insertDirectly(@NotNull Novelty.Accessor novelty, final int pos, @NotNull byte[] key, Object child); + + IPage mergeWithChildren(@NotNull Novelty.Accessor novelty); // del? + + IPage split(@NotNull Novelty.Accessor novelty, int from, int length); + + // save + + void flush(@NotNull Novelty.Accessor novelty); +} diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/InternalPage.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/InternalPage.java index 1e0ca5cc3f975522dda6c77b9f9f9efb0844009b..6ea2bc416d6c18d8a6a19191068f12da9930e0eb 100644 --- a/platform/on-air-index/src/com/intellij/platform/onair/tree/InternalPage.java +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/InternalPage.java @@ -2,6 +2,7 @@ package com.intellij.platform.onair.tree; import com.intellij.platform.onair.storage.api.*; +import com.intellij.platform.onair.tree.functional.BaseTransientPage; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -9,8 +10,10 @@ import java.io.PrintStream; import java.util.Arrays; import static com.intellij.platform.onair.tree.BTree.BYTES_PER_ADDRESS; +import static com.intellij.platform.onair.tree.StoredBTreeUtil.indent; +import static com.intellij.platform.onair.tree.StoredBTreeUtil.setChild; -public class InternalPage extends BasePage { +public class InternalPage extends BasePage implements IInternalPage { public InternalPage(byte[] backingArray, BTree tree, Address address, int size) { super(backingArray, tree, address, size); @@ -18,16 +21,16 @@ public class InternalPage extends BasePage { @Override @Nullable - protected byte[] get(@NotNull Novelty.Accessor novelty, @NotNull byte[] key) { - final int index = binarySearch(key); + public byte[] get(@NotNull Novelty.Accessor novelty, @NotNull byte[] key) { + final int index = BTreeCommon.binarySearch(backingArray, size, tree.getKeySize(), BYTES_PER_ADDRESS, key); + return index < 0 ? getChild(novelty, Math.max(-index - 2, 0)).get(novelty, key) : getChild(novelty, index).get(novelty, key); } @Override - protected boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull KeyValueConsumer consumer) { + public boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull KeyValueConsumer consumer) { for (int i = 0; i < size; i++) { - Address childAddress = getChildAddress(i); - BasePage child = tree.loadPage(novelty, childAddress); + BasePage child = getChild(novelty, i); if (!child.forEach(novelty, consumer)) { return false; } @@ -36,29 +39,16 @@ public class InternalPage extends BasePage { } @Override - protected boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull byte[] fromKey, @NotNull KeyValueConsumer consumer) { - boolean first = true; - for (int i = binarySearchGuess(fromKey); i < size; i++) { - Address childAddress = getChildAddress(i); - BasePage child = tree.loadPage(novelty, childAddress); - if (first) { - if (!child.forEach(novelty, fromKey, consumer)) { - return false; - } - first = false; - } else { - if (!child.forEach(novelty, consumer)) { - return false; - } - } - } - return true; + public boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull byte[] fromKey, @NotNull KeyValueConsumer consumer) { + final int fromIndex = BTreeCommon.binarySearchGuess(backingArray, size, tree.getKeySize(), BYTES_PER_ADDRESS, fromKey); + + return BTreeCommon.traverseInternalPage(this, novelty, fromIndex, fromKey, consumer); } @Override @Nullable - protected BasePage put(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @NotNull byte[] value, boolean overwrite, boolean[] result) { - int pos = binarySearch(key); + public BasePage put(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @NotNull byte[] value, boolean overwrite, boolean[] result) { + int pos = BTreeCommon.binarySearch(backingArray, size, tree.getKeySize(), BYTES_PER_ADDRESS, key); if (pos >= 0 && !overwrite) { // key found and overwrite is not possible - error @@ -71,22 +61,16 @@ public class InternalPage extends BasePage { if (pos < 0) pos = 0; } - final BasePage child = getChild(novelty, pos).getMutableCopy(novelty, tree); + final BasePage child = getChild(novelty, pos).getMutableCopy(novelty); final BasePage newChild = child.put(novelty, key, value, overwrite, result); // change min key for child if (result[0]) { - if (!child.address.isNovelty()) { - throw new IllegalStateException("child must be novelty"); - } - set(pos, child.getMinKey(), child.address.getLowBytes()); + set(pos, child.getMinKey(), child); if (newChild == null) { flush(novelty); } else { - if (!newChild.address.isNovelty()) { - throw new IllegalStateException("child must be novelty"); - } - return insertAt(novelty, pos + 1, newChild.getMinKey(), newChild.address.getLowBytes()); + return BTreeCommon.insertAt(this, tree.getBase(), novelty, pos + 1, newChild.getMinKey(), newChild); } } @@ -94,35 +78,35 @@ public class InternalPage extends BasePage { } @Override - protected boolean delete(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @Nullable byte[] value) { - int pos = binarySearchGuess(key); - final BasePage child = getChild(novelty, pos).getMutableCopy(novelty, tree); + public boolean delete(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @Nullable byte[] value) { + int pos = BTreeCommon.binarySearchGuess(backingArray, size, tree.getKeySize(), BYTES_PER_ADDRESS, key); + final BasePage child = getChild(novelty, pos).getMutableCopy(novelty); if (!child.delete(novelty, key, value)) { return false; } // if first element was removed in child, then update min key final int childSize = child.size; if (childSize > 0) { - set(pos, child.getMinKey(), child.address.getLowBytes()); + set(pos, child.getMinKey(), child); } if (pos > 0) { final BasePage left = getChild(novelty, pos - 1); - if (needMerge(left, child)) { + if (BTreeCommon.needMerge(left, child, tree.getBase())) { // merge child into left sibling // re-get mutable left - getChild(novelty, pos - 1).getMutableCopy(novelty, tree).mergeWith(child); + getChild(novelty, pos - 1).getMutableCopy(novelty).mergeWith(child); removeChild(pos); } } else if (pos + 1 < size) { final BasePage right = getChild(novelty, pos + 1); - if (needMerge(child, right)) { + if (BTreeCommon.needMerge(child, right, tree.getBase())) { // merge child with right sibling - final BasePage mutableChild = child.getMutableCopy(novelty, tree); + final BasePage mutableChild = child.getMutableCopy(novelty); mutableChild.mergeWith(getChild(novelty, pos + 1)); removeChild(pos); // change key for link to right - set(pos, mutableChild.getMinKey(), mutableChild.address.getLowBytes()); + set(pos, mutableChild.getMinKey(), mutableChild); } } else if (childSize == 0) { @@ -133,14 +117,14 @@ public class InternalPage extends BasePage { } @Override - protected BasePage split(@NotNull Novelty.Accessor novelty, int from, int length) { - final InternalPage result = copyOf(novelty, this, from, length); + public IPage split(@NotNull Novelty.Accessor novelty, int from, int length) { + final IPage result = copyOf(novelty, this, from, length); decrementSize(length); return result; } @Override - protected InternalPage getMutableCopy(@NotNull Novelty.Accessor novelty, BTree tree) { + protected InternalPage getMutableCopy(@NotNull Novelty.Accessor novelty) { if (tree.canMutateInPlace(address)) { return this; } @@ -151,6 +135,11 @@ public class InternalPage extends BasePage { ); } + @Override + public BaseTransientPage getTransientCopy(long epoch) { + throw new UnsupportedOperationException(); // TODO + } + @Override protected Address save(@NotNull Novelty.Accessor novelty, @NotNull Storage storage, @NotNull StorageConsumer consumer) { final byte[] resultBytes = Arrays.copyOf(backingArray, backingArray.length); @@ -169,26 +158,31 @@ public class InternalPage extends BasePage { @Override @NotNull - protected BasePage getChild(@NotNull Novelty.Accessor novelty, final int index) { + public BasePage getChild(@NotNull Novelty.Accessor novelty, final int index) { return tree.loadPage(novelty, getChildAddress(index)); } @Override - protected BasePage mergeWithChildren(@NotNull Novelty.Accessor novelty) { + public BasePage mergeWithChildren(@NotNull Novelty.Accessor novelty) { BasePage result = this; - while (!result.isBottom() && result.size == 1) { - result = result.getChild(novelty, 0); + while (!result.isBottom() && result.getSize() == 1) { + result = ((InternalPage)result).getChild(novelty, 0); } return result; } - protected void removeChild(int pos) { - copyChildren(pos + 1, pos); - decrementSize(1); + @Override + public void insertDirectly(@NotNull Novelty.Accessor novelty, final int pos, @NotNull byte[] key, Object child) { + if (pos < size) { + copyChildren(pos, pos + 1); + } + set(pos, key, (IPage)child); + incrementSize(); + flush(novelty); } @Override - protected boolean isBottom() { + public boolean isBottom() { return false; } @@ -205,6 +199,33 @@ public class InternalPage extends BasePage { } } + private void removeChild(int pos) { + copyChildren(pos + 1, pos); + decrementSize(1); + } + + private void set(int pos, byte[] key, IPage child) { + final int bytesPerKey = tree.getKeySize(); + + if (key.length != bytesPerKey) { + throw new IllegalArgumentException("Invalid key length: need " + bytesPerKey + ", got: " + key.length); + } + + StoredBTreeUtil.set(pos, key, bytesPerKey, backingArray, child.getMutableAddress()); + } + + private void copyChildren(final int from, final int to) { + if (from >= size) return; + + final int bytesPerEntry = tree.getKeySize() + BYTES_PER_ADDRESS; + + System.arraycopy( + backingArray, from * bytesPerEntry, + backingArray, to * bytesPerEntry, + (size - from) * bytesPerEntry + ); + } + private static InternalPage copyOf(@NotNull Novelty.Accessor novelty, InternalPage page, int from, int length) { byte[] bytes = new byte[page.backingArray.length]; diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/StoredBTreeUtil.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/StoredBTreeUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..e500818a8a4956256efd98f6779be8097c706ec3 --- /dev/null +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/StoredBTreeUtil.java @@ -0,0 +1,67 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.platform.onair.tree; + +import com.intellij.platform.onair.storage.api.Novelty; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.PrintStream; + +import static com.intellij.platform.onair.tree.BTree.BYTES_PER_ADDRESS; +import static com.intellij.platform.onair.tree.ByteUtils.writeUnsignedLong; + +public class StoredBTreeUtil { + public static BasePage delete(@NotNull Novelty.Accessor novelty, + @NotNull BasePage root, + @NotNull byte[] key, + @Nullable byte[] value, + boolean[] res) { + if (root.delete(novelty, key, value)) { + root = root.mergeWithChildren(novelty); + res[0] = true; + return root; + } + + res[0] = false; + return root; + } + + public static void set(int pos, byte[] key, int bytesPerKey, byte[] backingArray, long lowAddressBytes) { + final int offset = (bytesPerKey + BYTES_PER_ADDRESS) * pos; + + // write key + System.arraycopy(key, 0, backingArray, offset, bytesPerKey); + // write address + writeUnsignedLong(lowAddressBytes, 8, backingArray, offset + bytesPerKey); + writeUnsignedLong(0, 8, backingArray, offset + bytesPerKey + 8); + } + + public static void set(int pos, byte[] key, int bytesPerKey, byte[] backingArray, byte[] inlineValue) { + int offset = (bytesPerKey + BYTES_PER_ADDRESS) * pos; + + // write key + System.arraycopy(key, 0, backingArray, offset, bytesPerKey); + // write value + offset += bytesPerKey; + System.arraycopy(inlineValue, 0, backingArray, offset, inlineValue.length); + backingArray[offset + BYTES_PER_ADDRESS - 1] = (byte)inlineValue.length; + } + + public static void setChild(int pos, int bytesPerKey, byte[] backingArray, long lowAddressBytes, long highAddressBytes) { + final int offset = (bytesPerKey + BYTES_PER_ADDRESS) * pos; + // write address + writeUnsignedLong(lowAddressBytes, 8, backingArray, offset + bytesPerKey); + writeUnsignedLong(highAddressBytes, 8, backingArray, offset + bytesPerKey + 8); + } + + public static void setChild(int pos, int bytesPerKey, byte[] backingArray, byte[] inlineValue) { + final int offset = (bytesPerKey + BYTES_PER_ADDRESS) * pos + bytesPerKey; + // write value + System.arraycopy(inlineValue, 0, backingArray, offset, inlineValue.length); + backingArray[offset + BYTES_PER_ADDRESS - 1] = (byte)inlineValue.length; + } + + public static void indent(PrintStream out, int level) { + for (int i = 0; i < level; i++) out.print(" "); + } +} diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/BaseTransientPage.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/BaseTransientPage.java new file mode 100644 index 0000000000000000000000000000000000000000..3871c2947a1c9a789c6b0103cb1f4b1f758bb21b --- /dev/null +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/BaseTransientPage.java @@ -0,0 +1,97 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.platform.onair.tree.functional; + +import com.intellij.platform.onair.storage.api.Novelty; +import com.intellij.platform.onair.tree.IPage; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Arrays; + +public abstract class BaseTransientPage implements IPage { + + protected final byte[] backingArray; // keys only + protected final TransientBTreePrototype tree; + protected final long epoch; + + protected int size; // TODO: allow in-place update only for current epoch nodes + + protected BaseTransientPage(byte[] backingArray, TransientBTreePrototype tree, int size, long epoch) { + this.backingArray = backingArray; + this.tree = tree; + this.size = size; + this.epoch = epoch; + } + + @Override + public int getSize() { + return size; + } + + @Override + public boolean isTransient() { + return true; + } + + @Override + public void flush(@NotNull Novelty.Accessor novelty) { + // do nothing + } + + @Override + public long getMutableAddress() { + throw new UnsupportedOperationException(); + } + + /*@Override + public abstract IPage mergeWithChildren(@NotNull Novelty.Accessor novelty);*/ + + @Override + @NotNull + public byte[] getMinKey() { + if (size <= 0) { + throw new ArrayIndexOutOfBoundsException("Page is empty."); + } + + return Arrays.copyOf(backingArray, tree.keySize); // TODO: optimize + } + + @Nullable + public abstract BaseTransientPage put(@NotNull Novelty.Accessor novelty, + long epoch, + @NotNull byte[] key, + @NotNull byte[] value, + boolean overwrite, + boolean[] result); + + public abstract boolean delete(@NotNull Novelty.Accessor novelty, long epoch, @NotNull byte[] key, @Nullable byte[] value); + + protected void incrementSize() { + if (size >= tree.base) { + throw new IllegalArgumentException("Can't increase tree page size"); + } + size += 1; + } + + protected void decrementSize(final int value) { + if (size < value) { + throw new IllegalArgumentException("Can't decrease tree page size " + size + " on " + value); + } + size -= value; + } + + // WARNING: this method allocates an array + protected byte[] getKey(int index) { + final int bytesPerKey = tree.keySize; + byte[] result = new byte[bytesPerKey]; + final int offset = bytesPerKey * index; + System.arraycopy(backingArray, offset, result, 0, bytesPerKey); + return result; + } + + protected void mergeWith(BaseTransientPage page) { + final int bytesPerKey = tree.keySize; + System.arraycopy(page.backingArray, 0, backingArray, size * bytesPerKey, page.size); + this.size += page.size; + } +} diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/BottomTransientPage.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/BottomTransientPage.java new file mode 100644 index 0000000000000000000000000000000000000000..96a7f59a3ce2562bd72aa16c6161d966b409050e --- /dev/null +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/BottomTransientPage.java @@ -0,0 +1,177 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.platform.onair.tree.functional; + +import com.intellij.platform.onair.storage.api.Address; +import com.intellij.platform.onair.storage.api.KeyValueConsumer; +import com.intellij.platform.onair.storage.api.Novelty; +import com.intellij.platform.onair.tree.BTreeCommon; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class BottomTransientPage extends BaseTransientPage { + protected final Object[] values; // Address | byte[] + + public BottomTransientPage(byte[] backingArray, TransientBTreePrototype tree, int size, long epoch, Object[] values) { + super(backingArray, tree, size, epoch); + this.values = values; + } + + @Nullable + @Override + public byte[] get(@NotNull Novelty.Accessor novelty, @NotNull byte[] key) { + final int index = BTreeCommon.binarySearch(backingArray, size, tree.keySize, 0, key); + if (index >= 0) { + return getValue(novelty, index); + } + return null; + } + + @Override + public boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull KeyValueConsumer consumer) { + for (int i = 0; i < size; i++) { + byte[] key = getKey(i); + byte[] value = getValue(novelty, i); + if (!consumer.consume(key, value)) { + return false; + } + } + return true; + } + + @Override + public boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull byte[] fromKey, @NotNull KeyValueConsumer consumer) { + for (int i = BTreeCommon.binarySearchRange(backingArray, size, tree.keySize, 0, fromKey); i < size; i++) { + byte[] key = getKey(i); + byte[] value = getValue(novelty, i); + if (!consumer.consume(key, value)) { + return false; + } + } + return true; + } + + @Nullable + @Override + public BaseTransientPage put(@NotNull Novelty.Accessor novelty, + long epoch, @NotNull byte[] key, + @NotNull byte[] value, + boolean overwrite, + boolean[] result) { + throw new UnsupportedOperationException(); // TODO + } + + @Override + public boolean delete(@NotNull Novelty.Accessor novelty, long epoch, @NotNull byte[] key, byte[] value) { + throw new UnsupportedOperationException(); // TODO + } + + @Override + public void insertDirectly(@NotNull Novelty.Accessor novelty, final int pos, @NotNull byte[] key, Object child) { + if (pos < size) { + copyChildren(pos, pos + 1); + } + + final int bytesPerKey = tree.keySize; + + if (key.length != bytesPerKey) { + throw new IllegalArgumentException("Invalid key length: need " + bytesPerKey + ", got: " + key.length); + } + + setTransient(pos, key, (byte[])child); + + incrementSize(); + flush(novelty); + } + + @Override + public BottomTransientPage split(@NotNull Novelty.Accessor novelty, int from, int length) { + final BottomTransientPage result = copyOf(this, epoch, from, length); + decrementSize(length); + flush(novelty); + return result; + } + + @Override + public BottomTransientPage getTransientCopy(long epoch) { + if (this.epoch >= epoch) { + return this; + } else { + return copyOf(this, epoch, 0, size); + } + } + + @Override + public BaseTransientPage mergeWithChildren(@NotNull Novelty.Accessor novelty) { + return this; + } + + @Override + public boolean isBottom() { + return true; + } + + private byte[] getValue(@NotNull Novelty.Accessor novelty, int index) { + final Object child = values[index]; + if (child instanceof byte[]) { + return (byte[])child; + } + final Address address = (Address)child; + final boolean isNovelty = address.isNovelty(); + return isNovelty ? novelty.lookup(address.getLowBytes()) : tree.storage.lookup(address); + } + + private void setTransient(int pos, byte[] key, byte[] child) { + final int bytesPerKey = tree.keySize; + final int offset = bytesPerKey * pos; + + // write key + System.arraycopy(key, 0, backingArray, offset, bytesPerKey); + + // write value + values[pos] = child; + } + + private void copyChildren(final int from, final int to) { + if (from >= size) return; + + final int bytesPerKey = tree.keySize; + + // copy keys + System.arraycopy( + backingArray, from * bytesPerKey, + backingArray, to * bytesPerKey, + (size - from) * bytesPerKey + ); + + // copy values + System.arraycopy( + values, from, + values, to, + (size - from) + ); + } + + private static BottomTransientPage copyOf(BottomTransientPage page, long epoch, int from, int length) { + byte[] bytes = new byte[page.backingArray.length]; + + final int bytesPerKey = page.tree.keySize; + + // copy keys + System.arraycopy( + page.backingArray, from * bytesPerKey, + bytes, 0, + length * bytesPerKey + ); + + Object[] values = new Object[page.values.length]; + + // copy values + System.arraycopy( + page.values, from, + values, 0, + length + ); + + return new BottomTransientPage(bytes, page.tree, length, epoch, values); + } +} diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/InternalTransientPage.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/InternalTransientPage.java new file mode 100644 index 0000000000000000000000000000000000000000..7a7ed7023159ef61b424b7603645889475be6b45 --- /dev/null +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/InternalTransientPage.java @@ -0,0 +1,245 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.platform.onair.tree.functional; + +import com.intellij.platform.onair.storage.api.Address; +import com.intellij.platform.onair.storage.api.KeyValueConsumer; +import com.intellij.platform.onair.storage.api.Novelty; +import com.intellij.platform.onair.tree.BTreeCommon; +import com.intellij.platform.onair.tree.IInternalPage; +import com.intellij.platform.onair.tree.IPage; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class InternalTransientPage extends BaseTransientPage implements IInternalPage { + + protected final IPage[] children; // Address | IPage + + public InternalTransientPage(byte[] backingArray, TransientBTreePrototype tree, int size, long epoch, IPage[] children) { + super(backingArray, tree, size, epoch); + this.children = children; + } + + @Override + @Nullable + public byte[] get(@NotNull Novelty.Accessor novelty, @NotNull byte[] key) { + final int index = BTreeCommon.binarySearch(backingArray, size, tree.keySize, 0, key); + + return index < 0 ? getChild(novelty, Math.max(-index - 2, 0)).get(novelty, key) : getChild(novelty, index).get(novelty, key); + } + + @Override + public boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull KeyValueConsumer consumer) { + for (int i = 0; i < size; i++) { + IPage child = getChild(novelty, i); + if (!child.forEach(novelty, consumer)) { + return false; + } + } + return true; + } + + @Override + public boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull byte[] fromKey, @NotNull KeyValueConsumer consumer) { + final int fromIndex = BTreeCommon.binarySearchGuess(backingArray, size, tree.keySize, 0, fromKey); + + return BTreeCommon.traverseInternalPage(this, novelty, fromIndex, fromKey, consumer); + } + + @Override + @Nullable + public BaseTransientPage put(@NotNull Novelty.Accessor novelty, + long epoch, @NotNull byte[] key, + @NotNull byte[] value, + boolean overwrite, + boolean[] result) { + int pos = BTreeCommon.binarySearch(backingArray, size, tree.keySize, 0, key); + + if (pos >= 0 && !overwrite) { + // key found and overwrite is not possible - error + return null; + } + + if (pos < 0) { + pos = -pos - 2; + // if insert after last - set to last + if (pos < 0) pos = 0; + } + + final BaseTransientPage child = getChild(novelty, pos).getTransientCopy(epoch); + final IPage newChild = child.put(novelty, epoch, key, value, overwrite, result); + // change min key for child + if (result[0]) { + setTransient(pos, child.getMinKey(), child); + if (newChild == null) { + flush(novelty); + } + else { + return BTreeCommon.insertAt(this, tree.base, novelty, pos + 1, newChild.getMinKey(), newChild); + } + } + + return null; + } + + @Override + public boolean delete(@NotNull Novelty.Accessor novelty, long epoch, @NotNull byte[] key, @Nullable byte[] value) { + int pos = BTreeCommon.binarySearchGuess(backingArray, size, tree.keySize, 0, key); + final BaseTransientPage child = getChild(novelty, pos).getTransientCopy(epoch); + if (!child.delete(novelty, epoch, key, value)) { + return false; + } + // if first element was removed in child, then update min key + final int childSize = child.getSize(); + if (childSize > 0) { + setTransient(pos, child.getMinKey(), child); + } + if (pos > 0) { + final IPage left = getChild(novelty, pos - 1); + if (BTreeCommon.needMerge(left, child, tree.base)) { + // merge child into left sibling + // re-get mutable left + getChild(novelty, pos - 1).getTransientCopy(epoch).mergeWith(child); + removeChild(pos); + } + } + else if (pos + 1 < size) { + final IPage right = getChild(novelty, pos + 1); + if (BTreeCommon.needMerge(child, right, tree.base)) { + // merge child with right sibling + final BaseTransientPage mutableChild = child.getTransientCopy(epoch); + IPage sibling = getChild(novelty, pos + 1); + mutableChild.mergeWith(sibling.getTransientCopy(epoch)); + removeChild(pos); + // change key for link to right + setTransient(pos, mutableChild.getMinKey(), mutableChild); + } + } + else if (childSize == 0) { + removeChild(pos); + } + return true; + } + + @Override + public InternalTransientPage getTransientCopy(long epoch) { + if (this.epoch >= epoch) { + return this; + } + else { + return copyOf(this, epoch, 0, size); + } + } + + @Override + public IPage split(@NotNull Novelty.Accessor novelty, int from, int length) { + final IPage result = copyOf(this, epoch, from, length); + decrementSize(length); + return result; + } + + @Override + @NotNull + public IPage getChild(@NotNull Novelty.Accessor novelty, final int index) { + final Object child = children[index]; + if (child instanceof BaseTransientPage) { + return (BaseTransientPage)child; + } + return tree.storedTree.loadPage(novelty, (Address)child); + } + + @Override + public void insertDirectly(@NotNull Novelty.Accessor novelty, final int pos, @NotNull byte[] key, Object child) { + if (pos < size) { + copyChildren(pos, pos + 1); + } + final IPage page = (IPage)child; + if (page.isTransient()) { + setTransient(pos, key, page); + } + else { + throw new IllegalArgumentException("non-transient child cannot be inserted here"); + } + incrementSize(); + flush(novelty); + } + + @Override + public boolean isBottom() { + return false; + } + + @Override + public IPage mergeWithChildren(@NotNull Novelty.Accessor novelty) { + IPage result = this; + while (!result.isBottom() && result.getSize() == 1) { + result = ((IInternalPage)result).getChild(novelty, 0); + } + return result; + } + + @Override + protected void mergeWith(BaseTransientPage page) { + System.arraycopy(((InternalTransientPage)page).children, 0, children, size, page.size); + super.mergeWith(page); + } + + private void removeChild(int pos) { + copyChildren(pos + 1, pos); + decrementSize(1); + } + + private void setTransient(int pos, byte[] key, IPage child) { + final int bytesPerKey = tree.keySize; + final int offset = bytesPerKey * pos; + + // write key + System.arraycopy(key, 0, backingArray, offset, bytesPerKey); + + // write value + children[pos] = child; + } + + private void copyChildren(final int from, final int to) { + if (from >= size) return; + + final int bytesPerKey = tree.keySize; + + // copy keys + System.arraycopy( + backingArray, from * bytesPerKey, + backingArray, to * bytesPerKey, + (size - from) * bytesPerKey + ); + + // copy children + System.arraycopy( + children, from, + children, to, + (size - from) + ); + } + + private static InternalTransientPage copyOf(InternalTransientPage page, long epoch, int from, int length) { + byte[] bytes = new byte[page.backingArray.length]; + + final int bytesPerKey = page.tree.keySize; + + // copy keys + System.arraycopy( + page.backingArray, from * bytesPerKey, + bytes, 0, + length * bytesPerKey + ); + + IPage[] children = new IPage[page.children.length]; + + // copy children + System.arraycopy( + page.children, from, + children, 0, + length + ); + + return new InternalTransientPage(bytes, page.tree, length, epoch, children); + } +} diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/TransientBTree.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/TransientBTree.java new file mode 100644 index 0000000000000000000000000000000000000000..9f44dbba909ba8f2d7f30e6f75bcecf2a3a46078 --- /dev/null +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/TransientBTree.java @@ -0,0 +1,128 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.platform.onair.tree.functional; + +import com.intellij.platform.onair.storage.api.*; +import com.intellij.platform.onair.tree.BTree; +import com.intellij.platform.onair.tree.BasePage; +import com.intellij.platform.onair.tree.IPage; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class TransientBTree implements TransientTree { + + final TransientBTreePrototype prototype; + + private final Object root; + private final long epoch; + + public TransientBTree(TransientBTreePrototype prototype, Object root, long epoch) { + this.prototype = prototype; + this.root = root; + this.epoch = epoch; + } + + @Override + public int getKeySize() { + return prototype.keySize; + } + + @Override + public int getBase() { + return BTree.DEFAULT_BASE; + } + + @Nullable + @Override + public byte[] get(@NotNull Novelty.Accessor novelty, @NotNull byte[] key) { + return root(novelty).get(novelty, key); + } + + @Override + public boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull KeyValueConsumer consumer) { + return root(novelty).forEach(novelty, consumer); + } + + @Override + public boolean forEach(@NotNull Novelty.Accessor novelty, @NotNull byte[] fromKey, @NotNull KeyValueConsumer consumer) { + return root(novelty).forEach(novelty, fromKey, consumer); + } + + @Override + public TransientTree put(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @NotNull byte[] value) { + return put(novelty, key, value, true); + } + + @Override + public TransientTree put(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @NotNull byte[] value, boolean overwrite) { + final boolean[] result = new boolean[1]; + final BaseTransientPage root = root(novelty).getTransientCopy(epoch); + final BaseTransientPage newSibling = root.put(novelty, epoch, key, value, overwrite, result); + final BaseTransientPage finalRoot; + if (newSibling != null) { + final byte[] bytes = new byte[getKeySize() * getBase()]; + final IPage[] children = new IPage[getBase()]; + final InternalTransientPage internalRoot = new InternalTransientPage(bytes, prototype, 2, epoch, children); + TransientBTreeUtil.set(0, root.getMinKey(), getKeySize(), bytes); + children[0] = root; + TransientBTreeUtil.set(1, newSibling.getMinKey(), getKeySize(), bytes); + children[1] = newSibling; + finalRoot = internalRoot; + } + else { + finalRoot = root; + } + return new TransientBTree(prototype, finalRoot, epoch); + } + + @Override + public TransientTree delete(@NotNull Novelty.Accessor novelty, @NotNull byte[] key) { + return delete(novelty, key, null); + } + + @Override + public TransientTree delete(@NotNull Novelty.Accessor novelty, @NotNull byte[] key, @Nullable byte[] value) { + final BaseTransientPage root = root(novelty).getTransientCopy(epoch); + final IPage updatedRoot = TransientBTreeUtil.delete(novelty, epoch, root, key, value); + if (root == updatedRoot) { + return this; + } + else { + // updatedRoot can be "downgraded" to stored page if some merge occurs + Object finalRoot = updatedRoot.isTransient() ? updatedRoot : ((BasePage)updatedRoot).getAddress(); + return new TransientBTree(prototype, finalRoot, epoch); + } + } + + @Override + public TransientTree flush() { + if (!(root instanceof IPage)) { + return this; // already on disk + } + + // TODO: flush pages + + return new TransientBTree(prototype, root, epoch + 1); + } + + @NotNull + private IPage root(@NotNull Novelty.Accessor novelty) { + if (root instanceof Address) { + return loadRootPage(novelty); + } + else { + return (IPage)root; + } + } + + private BasePage loadRootPage(@NotNull Novelty.Accessor novelty) { + final long startAddress; + final Address rootAddress = (Address)root; + if (rootAddress.isNovelty()) { + startAddress = rootAddress.getLowBytes(); + } + else { + startAddress = Long.MIN_VALUE; + } + return new BTree(prototype.storage, prototype.keySize, rootAddress, startAddress).loadPage(novelty, rootAddress); + } +} diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/TransientBTreePrototype.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/TransientBTreePrototype.java new file mode 100644 index 0000000000000000000000000000000000000000..7aee97e7a11028e150d1e01e053f4903299c5562 --- /dev/null +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/TransientBTreePrototype.java @@ -0,0 +1,19 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.platform.onair.tree.functional; + +import com.intellij.platform.onair.storage.api.Storage; +import com.intellij.platform.onair.tree.BTree; + +/*package*/ class TransientBTreePrototype { + final BTree storedTree; + final Storage storage; + final int keySize; + final int base; + + /*package*/ TransientBTreePrototype(BTree storedTree, Storage storage) { + this.storedTree = storedTree; + this.storage = storage; + this.keySize = storedTree.getKeySize(); + this.base = storedTree.getBase(); + } +} diff --git a/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/TransientBTreeUtil.java b/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/TransientBTreeUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..2b5ebf445a7ddd4e8b34097c2c76a34b8f848b6e --- /dev/null +++ b/platform/on-air-index/src/com/intellij/platform/onair/tree/functional/TransientBTreeUtil.java @@ -0,0 +1,29 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.platform.onair.tree.functional; + +import com.intellij.platform.onair.storage.api.Novelty; +import com.intellij.platform.onair.tree.IPage; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class TransientBTreeUtil { + + public static IPage delete(@NotNull Novelty.Accessor novelty, + long epoch, + @NotNull BaseTransientPage root, + @NotNull byte[] key, + @Nullable byte[] value) { + if (root.delete(novelty, epoch, key, value)) { + return root.mergeWithChildren(novelty); + } + + return root; + } + + public static void set(int pos, byte[] key, int bytesPerKey, byte[] backingArray) { + final int offset = bytesPerKey * pos; + + // write key + System.arraycopy(key, 0, backingArray, offset, bytesPerKey); + } +}