From e60867b85c55b436822594edac4253b6ac5ff9ac Mon Sep 17 00:00:00 2001 From: qaate47 Date: Wed, 10 Jan 2024 17:07:46 +0100 Subject: [PATCH] GH-440 Add delete bitmap to multiple co-indexes --- .../qendpoint/core/hdt/impl/HDTImpl.java | 10 ++ .../core/header/PlainHeaderIterator.java | 5 + .../iterator/DictionaryTranslateIterator.java | 5 + .../DictionaryTranslateIteratorBuffer.java | 5 + .../qendpoint/core/options/HDTOptions.java | 74 +++++++++ .../core/rdf/parsers/JenaModelIterator.java | 4 + .../core/triples/IteratorTripleID.java | 12 +- .../core/triples/IteratorTripleString.java | 2 + .../triples/impl/BitmapTriplesIterator.java | 5 + .../qendpoint/store/EndpointFiles.java | 25 +-- .../qendpoint/store/EndpointStore.java | 142 ++++++++++------ .../store/EndpointStoreConnection.java | 34 ++-- .../store/EndpointStoreTripleIterator.java | 5 +- .../qendpoint/store/MergeRunnable.java | 38 +++-- .../qendpoint/tools/QEPSearch.java | 3 +- ...tMultIndexSPARQL11QueryComplianceTest.java | 153 ++++++++++++++++++ ...MultIndexSPARQL11UpdateComplianceTest.java | 102 ++++++++++++ .../qendpoint/store/MergeRestartTest.java | 18 --- .../qendpoint/store/Utility.java | 2 +- 19 files changed, 542 insertions(+), 102 deletions(-) create mode 100644 qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/EndpointMultIndexSPARQL11QueryComplianceTest.java create mode 100644 qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/EndpointMultIndexSPARQL11UpdateComplianceTest.java diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/HDTImpl.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/HDTImpl.java index 11e22692..b7d2c95c 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/HDTImpl.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/HDTImpl.java @@ -333,6 +333,11 @@ public long getLastTriplePosition() { public TripleComponentOrder getOrder() { return TripleComponentOrder.getAcceptableOrder(searchOrderMask); } + + @Override + public boolean isLastTriplePositionBoundToOrder() { + return false; + } }; } @@ -411,6 +416,11 @@ public long getLastTriplePosition() { public TripleComponentOrder getOrder() { return TripleComponentOrder.getAcceptableOrder(searchOrderMask); } + + @Override + public boolean isLastTriplePositionBoundToOrder() { + return false; + } }; } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/header/PlainHeaderIterator.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/header/PlainHeaderIterator.java index 5c0a6b35..a5f823ae 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/header/PlainHeaderIterator.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/header/PlainHeaderIterator.java @@ -116,4 +116,9 @@ public long getLastTriplePosition() { public TripleComponentOrder getOrder() { return TripleComponentOrder.Unknown; } + + @Override + public boolean isLastTriplePositionBoundToOrder() { + return false; + } } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/DictionaryTranslateIterator.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/DictionaryTranslateIterator.java index b26015b5..4d265a09 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/DictionaryTranslateIterator.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/DictionaryTranslateIterator.java @@ -167,4 +167,9 @@ public TripleComponentOrder getOrder() { return iterator.getOrder(); } + @Override + public boolean isLastTriplePositionBoundToOrder() { + return iterator.isLastTriplePositionBoundToOrder(); + } + } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/DictionaryTranslateIteratorBuffer.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/DictionaryTranslateIteratorBuffer.java index 84553d64..d1da6302 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/DictionaryTranslateIteratorBuffer.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/iterator/DictionaryTranslateIteratorBuffer.java @@ -284,6 +284,11 @@ public TripleComponentOrder getOrder() { return iterator.getOrder(); } + @Override + public boolean isLastTriplePositionBoundToOrder() { + return iterator.isLastTriplePositionBoundToOrder(); + } + public static void setBlockSize(int size) { DEFAULT_BLOCK_SIZE = size; } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/options/HDTOptions.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/options/HDTOptions.java index 58c22deb..260947bf 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/options/HDTOptions.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/options/HDTOptions.java @@ -30,7 +30,9 @@ import java.io.Writer; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashSet; import java.util.Map; import java.util.Objects; @@ -41,6 +43,7 @@ import java.util.function.IntSupplier; import java.util.function.LongSupplier; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * Options storage, see {@link HDTOptionsKeys} for more information. @@ -488,6 +491,8 @@ default void set(String key, Object value) { set(key, fs); } else if (value instanceof Path p) { set(key, p.toAbsolutePath().toString()); + } else if (value instanceof EnumSet p) { + set(key, p); } else if (value instanceof File f) { set(key, f.getAbsolutePath()); } else { @@ -724,4 +729,73 @@ public Set getKeys() { } }; } + + /** + * set enum set, the elements are split using comas + * + * @param key key + * @param set set + */ + default void set(String key, EnumSet set) { + set(key, set.stream().map(Enum::name).collect(Collectors.joining(","))); + } + + /** + * get enum set from value, the elements are split using comas + * + * @param key key + * @param cls enum class + * @return enum set + * @param enum type + */ + default > EnumSet getEnumSet(String key, Class cls) { + return getEnumSet(key, cls, EnumSet.noneOf(cls)); + } + + /** + * get enum set from value, the elements are split using comas + * + * @param key key + * @param cls enum class + * @param defaultValue default value + * @return enum set + * @param enum type + */ + default > EnumSet getEnumSet(String key, Class cls, EnumSet defaultValue) { + return getEnumSet(key, cls, () -> defaultValue); + } + + /** + * get enum set from value, the elements are split using comas + * + * @param key key + * @param cls enum class + * @param defaultValue default value supplier + * @return enum set + * @param enum type + */ + default > EnumSet getEnumSet(String key, Class cls, Supplier> defaultValue) { + String val = get(key); + + if (val == null || val.isEmpty()) { + return defaultValue.get(); + } + + EnumSet set = EnumSet.noneOf(cls); + + String[] values = val.split(","); + + mainFor: + for (String value : values) { + for (E e : cls.getEnumConstants()) { + if (e.name().equalsIgnoreCase(value)) { + set.add(e); + continue mainFor; + } + } + throw new IllegalArgumentException("Bad option value: " + value); + } + + return set; + } } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/rdf/parsers/JenaModelIterator.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/rdf/parsers/JenaModelIterator.java index 1ca94353..49b0856f 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/rdf/parsers/JenaModelIterator.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/rdf/parsers/JenaModelIterator.java @@ -60,4 +60,8 @@ public TripleComponentOrder getOrder() { return TripleComponentOrder.Unknown; } + @Override + public boolean isLastTriplePositionBoundToOrder() { + return false; + } } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/IteratorTripleID.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/IteratorTripleID.java index b958ee62..828f58c7 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/IteratorTripleID.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/IteratorTripleID.java @@ -19,11 +19,11 @@ package com.the_qa_company.qendpoint.core.triples; -import java.util.Iterator; - import com.the_qa_company.qendpoint.core.enums.ResultEstimationType; import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; +import java.util.Iterator; + /** * Iterator of TripleID */ @@ -95,4 +95,12 @@ public interface IteratorTripleID extends Iterator { * @see Triples#findTriple(long) */ long getLastTriplePosition(); + + /** + * @return if the {@link #getLastTriplePosition()} function is returning an + * index based on {@link #getOrder()} or the triples order + */ + default boolean isLastTriplePositionBoundToOrder() { + return false; + } } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/IteratorTripleString.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/IteratorTripleString.java index f8798b96..8b10aef4 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/IteratorTripleString.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/IteratorTripleString.java @@ -63,4 +63,6 @@ public interface IteratorTripleString extends Iterator { * @return order of the components from the iterator */ TripleComponentOrder getOrder(); + + boolean isLastTriplePositionBoundToOrder(); } diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIterator.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIterator.java index 3118c149..248a0923 100644 --- a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIterator.java +++ b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIterator.java @@ -293,4 +293,9 @@ public void remove() { public long getLastTriplePosition() { return lastPosition; } + + @Override + public boolean isLastTriplePositionBoundToOrder() { + return true; + } } diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointFiles.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointFiles.java index 47faf14f..2bd35d55 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointFiles.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointFiles.java @@ -1,6 +1,7 @@ package com.the_qa_company.qendpoint.store; import com.the_qa_company.qendpoint.compiler.ParsedStringValue; +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; import com.the_qa_company.qendpoint.core.hdt.HDTVersion; import java.io.File; @@ -226,27 +227,33 @@ public String getHDTNewIndexV11() { * @return the delete triple * {@link com.the_qa_company.qendpoint.utils.BitArrayDisk} file */ - @ParsedStringValue("store.deleteBitmap") - public String getTripleDeleteArr() { - return this.locationHdt + "triples-delete.arr"; + public String getTripleDeleteArr(TripleComponentOrder order) { + if (order == TripleComponentOrder.SPO) { + return this.locationHdt + "triples-delete.arr"; + } + return this.locationHdt + "triples-delete-" + order.name().toLowerCase() + ".arr"; } /** * @return the temp delete triple * {@link com.the_qa_company.qendpoint.utils.BitArrayDisk} file */ - @ParsedStringValue("store.tempDeleteBitmap") - public String getTripleDeleteTempArr() { - return this.locationHdt + "triples-delete-temp.arr"; + public String getTripleDeleteTempArr(TripleComponentOrder order) { + if (order == TripleComponentOrder.SPO) { + return this.locationHdt + "triples-delete-temp.arr"; + } + return this.locationHdt + "triples-delete-" + order.name().toLowerCase() + "-temp.arr"; } /** * @return the copy delete triple * {@link com.the_qa_company.qendpoint.utils.BitArrayDisk} file */ - @ParsedStringValue("store.tempDeleteBitmapCopy") - public String getTripleDeleteCopyArr() { - return this.locationHdt + "triples-delete-cpy.arr"; + public String getTripleDeleteCopyArr(TripleComponentOrder order) { + if (order == TripleComponentOrder.SPO) { + return this.locationHdt + "triples-delete-cpy.arr"; + } + return this.locationHdt + "triples-delete-" + order.name().toLowerCase() + "-cpy.arr"; } /** diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java index 7316d4c5..d5be2558 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStore.java @@ -1,5 +1,6 @@ package com.the_qa_company.qendpoint.store; +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; import com.the_qa_company.qendpoint.core.enums.TripleComponentRole; import com.the_qa_company.qendpoint.core.exceptions.NotFoundException; import com.the_qa_company.qendpoint.core.exceptions.ParserException; @@ -57,6 +58,7 @@ import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.Path; +import java.util.EnumSet; import java.util.Iterator; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; @@ -95,10 +97,10 @@ public class EndpointStore extends AbstractNotifyingSail { // location of the native store // bitmap to mark which triples in HDT were deleted - private BitArrayDisk deleteBitMap; + private final BitArrayDisk[] deleteBitMap = new BitArrayDisk[TripleComponentOrder.values().length]; // bitmap used to mark deleted triples in HDT during a merge operation // FIXME: is this needed? - private BitArrayDisk tempdeleteBitMap; + private final BitArrayDisk[] tempdeleteBitMap = new BitArrayDisk[TripleComponentOrder.values().length]; // setting to put the delete map only in memory, i.e don't write to disk private final boolean inMemDeletes; private final boolean loadIntoMemory; @@ -145,6 +147,7 @@ public class EndpointStore extends AbstractNotifyingSail { private MergeRunnable.MergeThread mergerThread; private final AtomicReference dump = new AtomicReference<>(); private final AtomicBoolean dumping = new AtomicBoolean(); + private final EnumSet validOrders; public void deleteNativeLocks() throws IOException { // remove lock files of a hard shutdown (SAIL is already locked by @@ -163,6 +166,9 @@ public EndpointStore(EndpointFiles files, HDTOptions spec, boolean inMemDeletes, throws IOException { // load HDT file this.spec = (spec = HDTOptions.ofNullable(spec)); + validOrders = getHDTSpec().getEnumSet(HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, TripleComponentOrder.class); + validOrders.add(TripleComponentOrder.SPO); // we need at least SPO + debugId = ENDPOINT_DEBUG_ID_GEN.incrementAndGet(); EndpointStoreUtils.openEndpoint(this); this.endpointFiles = files; @@ -343,12 +349,16 @@ public void setThreshold(int threshold) { // init the delete array upon the first start of the store private void initDeleteArray() throws IOException { - if (this.inMemDeletes) - setDeleteBitMap(new BitArrayDisk(this.hdt.getTriples().getNumberOfElements())); - else { + if (this.inMemDeletes) { + for (TripleComponentOrder order : validOrders) { + setDeleteBitMap(order, new BitArrayDisk(this.hdt.getTriples().getNumberOfElements())); + } + } else { // @todo: these should be recovered from the file if it is there - setDeleteBitMap( - new BitArrayDisk(this.hdt.getTriples().getNumberOfElements(), endpointFiles.getTripleDeleteArr())); + for (TripleComponentOrder order : validOrders) { + setDeleteBitMap(order, new BitArrayDisk(this.hdt.getTriples().getNumberOfElements(), + endpointFiles.getTripleDeleteArr(order))); + } } } @@ -493,14 +503,14 @@ public HDT loadIndex() throws IOException { return HDTManager.loadIndexedHDT(endpointFiles.getHDTIndex(), null, spec); } else { // use disk implementation to generate the index if required - OverrideHDTOptions specOver = new OverrideHDTOptions(spec); - specOver.setOverride(HDTOptionsKeys.BITMAPTRIPLES_INDEX_METHOD_KEY, + HDTOptions top = spec.pushTop(); + top.set(HDTOptionsKeys.BITMAPTRIPLES_INDEX_METHOD_KEY, HDTOptionsKeys.BITMAPTRIPLES_INDEX_METHOD_VALUE_DISK); - specOver.setOverride(HDTOptionsKeys.BITMAPTRIPLES_SEQUENCE_DISK, true); - specOver.setOverride(HDTOptionsKeys.BITMAPTRIPLES_SEQUENCE_DISK_SUBINDEX, true); - specOver.setOverride(HDTOptionsKeys.BITMAPTRIPLES_SEQUENCE_DISK_LOCATION, + top.set(HDTOptionsKeys.BITMAPTRIPLES_SEQUENCE_DISK, true); + top.set(HDTOptionsKeys.BITMAPTRIPLES_SEQUENCE_DISK_SUBINDEX, true); + top.set(HDTOptionsKeys.BITMAPTRIPLES_SEQUENCE_DISK_LOCATION, endpointFiles.getLocationHdtPath().resolve("indexload").toAbsolutePath()); - return HDTManager.mapIndexedHDT(endpointFiles.getHDTIndex(), specOver, null); + return HDTManager.mapIndexedHDT(endpointFiles.getHDTIndex(), top, null); } } @@ -591,16 +601,26 @@ public void setHdtProps(HDTProps hdtProps) { this.hdtProps = hdtProps; } - public BitArrayDisk getDeleteBitMap() { + public BitArrayDisk getDeleteBitMap(TripleComponentOrder order) { + return deleteBitMap[order.ordinal()]; + } + + public BitArrayDisk[] getDeleteBitMaps() { return deleteBitMap; } - public void setDeleteBitMap(BitArrayDisk deleteBitMap) { - this.deleteBitMap = deleteBitMap; + public void setDeleteBitMap(TripleComponentOrder order, BitArrayDisk deleteBitMap) { + this.deleteBitMap[order.ordinal()] = deleteBitMap; } - public BitArrayDisk getTempDeleteBitMap() { - return tempdeleteBitMap; + public void setDeleteBitMap(BitArrayDisk[] deleteBitMaps) { + for (TripleComponentOrder order : validOrders) { + this.deleteBitMap[order.ordinal()] = deleteBitMaps[order.ordinal()]; + } + } + + public BitArrayDisk getTempDeleteBitMap(TripleComponentOrder order) { + return tempdeleteBitMap[order.ordinal()]; } public NTriplesWriter getRdfWriterTempTriples() { @@ -612,9 +632,15 @@ public NTriplesWriter getRdfWriterTempTriples() { * while merging */ public void initTempDeleteArray() throws IOException { - this.tempdeleteBitMap = new BitArrayDisk(this.hdt.getTriples().getNumberOfElements(), - endpointFiles.getTripleDeleteTempArr()); - this.tempdeleteBitMap.force(false); + for (TripleComponentOrder order : validOrders) { + this.tempdeleteBitMap[order.ordinal()] = new BitArrayDisk(this.hdt.getTriples().getNumberOfElements(), + endpointFiles.getTripleDeleteTempArr(order)); + } + for (BitArrayDisk b : this.tempdeleteBitMap) { + if (b != null) { + b.force(false); + } + } } /** @@ -641,7 +667,10 @@ public void initTempDump(boolean isRestarting) { public void resetDeleteArray(HDT newHdt) throws IOException { // delete array created at merge time - BitArrayDisk newDeleteArray = new BitArrayDisk(newHdt.getTriples().getNumberOfElements()); + BitArrayDisk[] newDeleteArray = new BitArrayDisk[TripleComponentOrder.values().length]; + for (TripleComponentOrder order : validOrders) { + newDeleteArray[order.ordinal()] = new BitArrayDisk(newHdt.getTriples().getNumberOfElements()); + } long lastOldSubject = -2; long lastNewSubject = -2; @@ -661,8 +690,9 @@ public void resetDeleteArray(HDT newHdt) throws IOException { // iterate over the temp array, convert the triples and mark it as // deleted in the new HDT file - for (long i = 0; i < tempdeleteBitMap.getNumBits(); i++) { - if (tempdeleteBitMap.access(i)) { // means that a triple has been + BitArrayDisk tempDeleteBitMap = getTempDeleteBitMap(TripleComponentOrder.SPO); + for (long i = 0; i < tempDeleteBitMap.getNumBits(); i++) { + if (tempDeleteBitMap.access(i)) { // means that a triple has been // deleted during merge // find the deleted triple in the old HDT index TripleID tripleID = this.hdt.getTriples().findTriple(i); @@ -722,35 +752,33 @@ public void resetDeleteArray(HDT newHdt) throws IOException { TripleID triple = new TripleID(subject, predicate, object); if (!triple.isNoMatch()) { - IteratorTripleID next = newHdt.getTriples().search(triple); - if (next.hasNext()) { - next.next(); - long newIndex = next.getLastTriplePosition(); - newDeleteArray.set(newIndex, true); + for (TripleComponentOrder sorder : validOrders) { + IteratorTripleID next = newHdt.getTriples().search(triple, sorder.mask); + if (next.hasNext()) { + assert next.getOrder() == sorder : "invalid order"; + next.next(); + long newIndex = next.getLastTriplePosition(); + newDeleteArray[sorder.ordinal()].set(newIndex, true); + } } } } } } - if (MergeRunnableStopPoint.debug) { - logger.debug("HDT cache saved element(s) ones={} in {}", tempdeleteBitMap.countOnes(), watch.stopAndShow()); - if (debugTotal != 0) { - logger.debug("debugSavedSubject : {} % | {} / {}", 100 * debugSavedSubject / debugTotal, - debugSavedSubject, debugTotal); - logger.debug("debugSavedPredicate : {} % | {} / {}", 100 * debugSavedPredicate / debugTotal, - debugSavedPredicate, debugTotal); - logger.debug("debugSavedObject : {} % | {} / {}", 100 * debugSavedObject / debugTotal, - debugSavedObject, debugTotal); - } else { - logger.debug("no remap"); + Closer.closeSingle(getDeleteBitMaps()); + try { + for (TripleComponentOrder sorder : validOrders) { + newDeleteArray[sorder.ordinal()].changeToInDisk(new File(endpointFiles.getTripleDeleteArr(sorder))); } - logger.debug("Tmp map: {}", tempdeleteBitMap.printInfo()); - logger.debug("New map: {}", newDeleteArray.printInfo()); + } catch (Throwable t) { + try { + Closer.closeSingle(newDeleteArray); + } catch (Throwable t2) { + t.addSuppressed(t2); + } + throw t; } - - getDeleteBitMap().close(); - newDeleteArray.changeToInDisk(new File(endpointFiles.getTripleDeleteArr())); this.setDeleteBitMap(newDeleteArray); } @@ -769,7 +797,13 @@ public void markDeletedTempTriples() throws IOException { search.next(); long index = search.getLastTriplePosition(); if (index >= 0) { - this.deleteBitMap.set(index, true); + TripleComponentOrder order; + if (search.isLastTriplePositionBoundToOrder()) { + order = search.getOrder(); + } else { + order = TripleComponentOrder.SPO; + } + this.deleteBitMap[order.ordinal()].set(index, true); } } } @@ -988,10 +1022,18 @@ public long countTriplesNativeStore() { } public void flushWrites() throws IOException { - getDeleteBitMap().force(true); + for (BitArrayDisk b : deleteBitMap) { + if (b != null) { + b.force(true); + } + } if (isMerging()) { getRdfWriterTempTriples().getWriter().flush(); - getTempDeleteBitMap().force(true); + for (BitArrayDisk b : tempdeleteBitMap) { + if (b != null) { + b.force(true); + } + } } logger.debug("Writes completed"); } @@ -1083,4 +1125,8 @@ public void setDumping(boolean val) { long getDebugId() { return debugId; } + + public EnumSet getValidOrders() { + return validOrders; + } } diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java index 703ef93b..f00d0ce9 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreConnection.java @@ -1,6 +1,7 @@ package com.the_qa_company.qendpoint.store; import com.the_qa_company.qendpoint.compiler.ConfigSailConnection; +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; import com.the_qa_company.qendpoint.store.exception.EndpointTimeoutException; import com.the_qa_company.qendpoint.utils.BitArrayDisk; import org.eclipse.rdf4j.common.concurrent.locks.Lock; @@ -435,7 +436,7 @@ protected long sizeInternal(Resource... contexts) throws SailException { long sizeNativeB = connB_read.size(contexts); long sizeHdt = this.endpoint.getHdt().getTriples().getNumberOfElements(); - long sizeDeleted = this.endpoint.getDeleteBitMap().countOnes(); + long sizeDeleted = this.endpoint.getDeleteBitMap(TripleComponentOrder.SPO).countOnes(); logger.info("---------------------------"); logger.info("Size native A:" + sizeNativeA); logger.info("Size native B:" + sizeNativeB); @@ -512,7 +513,10 @@ private boolean tripleExistInHDT(TripleID tripleID) { if (iter.hasNext()) { iter.next(); long index = iter.getLastTriplePosition(); - return !this.endpoint.getDeleteBitMap().access(index); + return !this.endpoint + .getDeleteBitMap( + iter.isLastTriplePositionBoundToOrder() ? iter.getOrder() : TripleComponentOrder.SPO) + .access(index); } return false; } @@ -520,17 +524,25 @@ private boolean tripleExistInHDT(TripleID tripleID) { private void assignBitMapDeletes(TripleID tripleID, Resource subj, IRI pred, Value obj) throws SailException { if (tripleID.getSubject() != -1 && tripleID.getPredicate() != -1 && tripleID.getObject() != -1) { - IteratorTripleID iter = endpoint.getHdt().getTriples().search(tripleID); + for (TripleComponentOrder order : endpoint.getValidOrders()) { + IteratorTripleID iter = endpoint.getHdt().getTriples().search(tripleID, order.mask); - if (iter.hasNext()) { - iter.next(); - long index = iter.getLastTriplePosition(); + if (iter.hasNext()) { + iter.next(); + long index = iter.getLastTriplePosition(); - if (!this.endpoint.getDeleteBitMap().access(index)) { - this.endpoint.getDeleteBitMap().set(index, true); - if (this.endpoint.isMerging()) - this.endpoint.getTempDeleteBitMap().set(index, true); - notifyStatementRemoved(this.endpoint.getValueFactory().createStatement(subj, pred, obj)); + assert iter.isLastTriplePositionBoundToOrder(); + TripleComponentOrder sorder = iter.getOrder(); + + if (!this.endpoint.getDeleteBitMap(sorder).access(index)) { + this.endpoint.getDeleteBitMap(sorder).set(index, true); + if (this.endpoint.isMerging()) { + this.endpoint.getTempDeleteBitMap(sorder).set(index, true); + } + if (order == TripleComponentOrder.SPO) { + notifyStatementRemoved(this.endpoint.getValueFactory().createStatement(subj, pred, obj)); + } + } } } } else { diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java index c8ae1d94..c2cf374e 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreTripleIterator.java @@ -1,5 +1,6 @@ package com.the_qa_company.qendpoint.store; +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; import com.the_qa_company.qendpoint.store.exception.EndpointTimeoutException; import org.eclipse.rdf4j.common.iteration.CloseableIteration; import org.eclipse.rdf4j.model.IRI; @@ -52,7 +53,9 @@ public boolean hasNext() { while (iterator.hasNext()) { TripleID tripleID = iterator.next(); long index = iterator.getLastTriplePosition(); - if (!endpoint.getDeleteBitMap().access(index)) { + TripleComponentOrder order = iterator.isLastTriplePositionBoundToOrder() ? iterator.getOrder() + : TripleComponentOrder.SPO; + if (!endpoint.getDeleteBitMap(order).access(index)) { Resource subject = endpoint.getHdtConverter().idToSubjectHDTResource(tripleID.getSubject()); IRI predicate = endpoint.getHdtConverter().idToPredicateHDTResource(tripleID.getPredicate()); Value object = endpoint.getHdtConverter().idToObjectHDTResource(tripleID.getObject()); diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/MergeRunnable.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/MergeRunnable.java index 6130a07c..6f5feed7 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/MergeRunnable.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/MergeRunnable.java @@ -1,6 +1,7 @@ package com.the_qa_company.qendpoint.store; import com.github.jsonldjava.shaded.com.google.common.base.Stopwatch; +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; import com.the_qa_company.qendpoint.store.exception.EndpointStoreException; import com.the_qa_company.qendpoint.utils.BitArrayDisk; import com.the_qa_company.qendpoint.utils.OverrideHDTOptions; @@ -559,10 +560,14 @@ private synchronized void step1(boolean restarting, Lock switchLock) throws Inte // a lock is needed here if (restarting) { // delete previous array in case of restart - Files.deleteIfExists(Path.of(endpointFiles.getTripleDeleteCopyArr())); + for (TripleComponentOrder order : endpoint.getValidOrders()) { + Files.deleteIfExists(Path.of(endpointFiles.getTripleDeleteCopyArr(order))); + } + } + for (TripleComponentOrder order : endpoint.getValidOrders()) { + Files.copy(Path.of(endpointFiles.getTripleDeleteArr(order)), + Path.of(endpointFiles.getTripleDeleteCopyArr(order)), StandardCopyOption.REPLACE_EXISTING); } - Files.copy(Path.of(endpointFiles.getTripleDeleteArr()), Path.of(endpointFiles.getTripleDeleteCopyArr()), - StandardCopyOption.REPLACE_EXISTING); EndpointStoreDump dumpInfo = endpoint.getDumpRef().getAndSet(null); // #391: stop the lucene updates @@ -622,7 +627,7 @@ private synchronized void step2(boolean restarting, Lock lock, EndpointStoreDump createHDTDump(endpointFiles.getRDFTempOutput(), endpointFiles.getHDTTempOutput()); // cat the original index and the temp index logger.debug("HDT Cat/Diff"); - catDiffIndexes(endpointFiles.getHDTIndex(), endpointFiles.getTripleDeleteCopyArr(), + catDiffIndexes(endpointFiles.getHDTIndex(), endpointFiles.getTripleDeleteCopyArr(TripleComponentOrder.SPO), endpointFiles.getHDTTempOutput(), endpointFiles.getHDTNewIndex()); logger.debug("CAT completed!!!!! " + endpointFiles.getLocationHdt()); @@ -638,8 +643,10 @@ private synchronized void step2(boolean restarting, Lock lock, EndpointStoreDump // deletes delete(endpointFiles.getRDFTempOutput()); delete(endpointFiles.getHDTTempOutput()); - delete(endpointFiles.getTripleDeleteCopyArr()); - delete(endpointFiles.getTripleDeleteArr()); + for (TripleComponentOrder order : endpoint.getValidOrders()) { + delete(endpointFiles.getTripleDeleteCopyArr(order)); + delete(endpointFiles.getTripleDeleteArr(order)); + } logger.info("End merge step 2"); @@ -662,7 +669,8 @@ private enum Step3SubStep { */ private Step3SubStep getStep3SubStep() { - boolean existsOldTripleDeleteTempArr = existsOld(endpointFiles.getTripleDeleteTempArr()); + boolean existsOldTripleDeleteTempArr = endpoint.getValidOrders().stream() + .anyMatch(order -> existsOld(endpointFiles.getTripleDeleteTempArr(order))); if (!exists(endpointFiles.getHDTNewIndexV11()) && existsOldTripleDeleteTempArr) { // after rename(endpointFiles.getHDTNewIndexV11(), @@ -693,7 +701,9 @@ private Step3SubStep getStep3SubStep() { * preload data for step 3 if restarting */ private void preloadStep3() { - deleteIfExists(endpointFiles.getTripleDeleteArr()); + for (TripleComponentOrder order : endpoint.getValidOrders()) { + deleteIfExists(endpointFiles.getTripleDeleteArr(order)); + } Step3SubStep step3SubStep = getStep3SubStep(); @@ -709,7 +719,9 @@ private void preloadStep3() { case AFTER_INDEX_OLD_RENAME: renameFromOld(endpointFiles.getHDTIndex()); case AFTER_TRIPLEDEL_TMP_OLD_RENAME: - renameFromOld(endpointFiles.getTripleDeleteTempArr()); + for (TripleComponentOrder order : endpoint.getValidOrders()) { + renameFromOld(endpointFiles.getTripleDeleteTempArr(order)); + } case BEFORE_ALL: break; } @@ -777,7 +789,9 @@ private synchronized void step3(boolean restarting, Lock lock, EndpointStoreDump // rename new hdt to old hdt name so that they are replaces // BEFORE_ALL - renameToOld(endpointFiles.getTripleDeleteTempArr()); + for (TripleComponentOrder order : endpoint.getValidOrders()) { + renameToOld(endpointFiles.getTripleDeleteTempArr(order)); + } // AFTER_TRIPLEDEL_TMP_OLD_RENAME renameToOld(endpointFiles.getHDTIndex()); debugStepPoint(MergeRunnableStopPoint.STEP3_FILES_MID1); @@ -810,7 +824,9 @@ private synchronized void step3(boolean restarting, Lock lock, EndpointStoreDump debugStepPoint(MergeRunnableStopPoint.STEP3_END); completedMerge(); // we've deleted the rename marker, we can delete the old HDT - deleteOld(endpointFiles.getTripleDeleteTempArr()); + for (TripleComponentOrder order : endpoint.getValidOrders()) { + deleteOld(endpointFiles.getTripleDeleteTempArr(order)); + } deleteOld(endpointFiles.getHDTIndex()); deleteOld(endpointFiles.getHDTIndexV11()); diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/tools/QEPSearch.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/tools/QEPSearch.java index f1018664..27e88932 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/tools/QEPSearch.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/tools/QEPSearch.java @@ -7,6 +7,7 @@ import com.the_qa_company.qendpoint.core.compact.sequence.SequenceLog64BigDisk; import com.the_qa_company.qendpoint.core.dictionary.DictionarySection; import com.the_qa_company.qendpoint.core.dictionary.impl.MultipleBaseDictionary; +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; import com.the_qa_company.qendpoint.core.enums.TripleComponentRole; import com.the_qa_company.qendpoint.core.exceptions.NotFoundException; import com.the_qa_company.qendpoint.core.exceptions.ParserException; @@ -623,7 +624,7 @@ public void executeSail(AbstractNotifyingSail store, Function valu size = ep.getHdt().getDictionary().getNobjects() - shared; } case "bitd" -> { - bm = ep.getDeleteBitMap(); + bm = ep.getDeleteBitMap(TripleComponentOrder.SPO); size = ep.getHdt().getTriples().getNumberOfElements(); } default -> { diff --git a/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/EndpointMultIndexSPARQL11QueryComplianceTest.java b/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/EndpointMultIndexSPARQL11QueryComplianceTest.java new file mode 100644 index 00000000..59caedf4 --- /dev/null +++ b/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/EndpointMultIndexSPARQL11QueryComplianceTest.java @@ -0,0 +1,153 @@ +package com.the_qa_company.qendpoint.store; + +import com.the_qa_company.qendpoint.core.enums.RDFNotation; +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; +import com.the_qa_company.qendpoint.core.exceptions.NotFoundException; +import com.the_qa_company.qendpoint.core.exceptions.ParserException; +import com.the_qa_company.qendpoint.core.hdt.HDT; +import com.the_qa_company.qendpoint.core.hdt.HDTManager; +import com.the_qa_company.qendpoint.core.options.HDTOptions; +import com.the_qa_company.qendpoint.core.options.HDTOptionsKeys; +import com.the_qa_company.qendpoint.core.triples.impl.BitmapTriplesIndexFile; +import org.eclipse.rdf4j.query.Dataset; +import org.eclipse.rdf4j.repository.Repository; +import org.eclipse.rdf4j.repository.sail.SailRepository; +import org.eclipse.rdf4j.sail.SailException; +import org.eclipse.rdf4j.testsuite.query.parser.sparql.manifest.SPARQL11QueryComplianceTest; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.JarURLConnection; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.stream.Stream; + +public class EndpointMultIndexSPARQL11QueryComplianceTest extends SPARQL11QueryComplianceTest { + private static final Logger logger = LoggerFactory.getLogger(EndpointSPARQL11QueryComplianceTest.class); + + public EndpointMultIndexSPARQL11QueryComplianceTest(String displayName, String testURI, String name, + String queryFileURL, String resultFileURL, Dataset dataset, boolean ordered, boolean laxCardinality) + throws ParserException, NotFoundException, IOException { + super(displayName, testURI, name, queryFileURL, resultFileURL, null, ordered, laxCardinality); + setUpHDT(dataset); + List testToIgnore = new ArrayList<>(); + // @todo these tests are failing and should not, they are skipped so + // that we can be sure that we see when + // currently passing tests are not failing. Many of these tests are not + // so problematic since we do not support + // named graphs anyway + testToIgnore.add("constructwhere02 - CONSTRUCT WHERE"); + testToIgnore.add("constructwhere03 - CONSTRUCT WHERE"); + testToIgnore.add("constructwhere04 - CONSTRUCT WHERE"); + testToIgnore.add("Exists within graph pattern"); + testToIgnore.add("(pp07) Path with one graph"); + testToIgnore.add("(pp35) Named Graph 2"); + testToIgnore.add("sq01 - Subquery within graph pattern"); + testToIgnore.add("sq02 - Subquery within graph pattern, graph variable is bound"); + testToIgnore.add("sq03 - Subquery within graph pattern, graph variable is not bound"); + testToIgnore.add("sq04 - Subquery within graph pattern, default graph does not apply"); + testToIgnore.add("sq05 - Subquery within graph pattern, from named applies"); + testToIgnore.add("sq06 - Subquery with graph pattern, from named applies"); + testToIgnore.add("sq07 - Subquery with from "); + testToIgnore.add("sq11 - Subquery limit per resource"); + testToIgnore.add("sq13 - Subqueries don't inject bindings"); + testToIgnore.add("sq14 - limit by resource"); + + this.setIgnoredTests(testToIgnore); + } + + @Rule + public TemporaryFolder tempDir = TemporaryFolder.builder().assureDeletion().build(); + + EndpointStore endpoint; + File nativeStore; + File hdtStore; + + @Override + protected Repository newRepository() throws Exception { + nativeStore = tempDir.newFolder(); + hdtStore = tempDir.newFolder(); + + HDTOptions spec = HDTOptions.of(HDTOptionsKeys.DICTIONARY_TYPE_KEY, + HDTOptionsKeys.DICTIONARY_TYPE_VALUE_MULTI_OBJECTS, HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, + EnumSet.of(TripleComponentOrder.SPO, TripleComponentOrder.OPS, TripleComponentOrder.PSO)); + Path fileName = Path.of(hdtStore.getAbsolutePath() + "/" + EndpointStoreTest.HDT_INDEX_NAME); + if (this.hdt == null) { + hdt = Utility.createTempHdtIndex(tempDir, true, false, spec); + } + assert hdt != null; + + hdt.saveToHDT(fileName, null); + + endpoint = new EndpointStore(hdtStore.getAbsolutePath() + "/", EndpointStoreTest.HDT_INDEX_NAME, spec, + nativeStore.getAbsolutePath() + "/", true) { + @Override + public HDT loadIndex() throws IOException { + HDT idx = super.loadIndex(); + if (idx.getTriples().getNumberOfElements() == 0) { + return idx; + } + try { + Path fileOPS = BitmapTriplesIndexFile.getIndexPath(fileName, TripleComponentOrder.OPS); + Assert.assertTrue("can't find " + fileOPS, Files.exists(fileOPS)); + Path filePSO = BitmapTriplesIndexFile.getIndexPath(fileName, TripleComponentOrder.PSO); + Assert.assertTrue("can't find " + filePSO, Files.exists(filePSO)); + } catch (Throwable t) { + try (Stream l = Files.list(fileName.getParent())) { + l.forEach(System.err::println); + } catch (Exception e) { + t.addSuppressed(e); + } + throw t; + } + return idx; + } + }; + // endpoint.setThreshold(2); + return new SailRepository(endpoint); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + } + + HDT hdt; + + private void setUpHDT(Dataset dataset) throws IOException, ParserException, NotFoundException { + if (dataset == null) { + return; + } + + String x = dataset.getDefaultGraphs().toString(); + if (x.equals("[]")) { + x = dataset.getNamedGraphs().toString(); + } + String str = x.substring(x.lastIndexOf("!") + 1).replace("]", ""); + + URL url = SPARQL11QueryComplianceTest.class.getResource(str); + File tmpDir = new File("test"); + if (tmpDir.mkdirs()) { + logger.debug("{} dir created.", tmpDir); + } + assert url != null; + JarURLConnection con = (JarURLConnection) url.openConnection(); + File file = new File(tmpDir, con.getEntryName()); + + HDTOptions spec = HDTOptions.of(); + + hdt = HDTManager.generateHDT(file.getAbsolutePath(), "http://www.example.org/", RDFNotation.guess(file), spec, + null); + assert hdt != null; + hdt.search("", "", "").forEachRemaining(System.out::println); + } +} diff --git a/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/EndpointMultIndexSPARQL11UpdateComplianceTest.java b/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/EndpointMultIndexSPARQL11UpdateComplianceTest.java new file mode 100644 index 00000000..698afb1c --- /dev/null +++ b/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/EndpointMultIndexSPARQL11UpdateComplianceTest.java @@ -0,0 +1,102 @@ +package com.the_qa_company.qendpoint.store; + +import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; +import com.the_qa_company.qendpoint.core.hdt.HDT; +import com.the_qa_company.qendpoint.core.options.HDTOptions; +import com.the_qa_company.qendpoint.core.options.HDTOptionsKeys; +import com.the_qa_company.qendpoint.core.triples.impl.BitmapTriplesIndex; +import com.the_qa_company.qendpoint.core.triples.impl.BitmapTriplesIndexFile; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.repository.Repository; +import org.eclipse.rdf4j.repository.sail.SailRepository; +import org.eclipse.rdf4j.sail.SailException; +import org.eclipse.rdf4j.testsuite.query.parser.sparql.manifest.SPARQL11UpdateComplianceTest; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +public class EndpointMultIndexSPARQL11UpdateComplianceTest extends SPARQL11UpdateComplianceTest { + + public EndpointMultIndexSPARQL11UpdateComplianceTest(String displayName, String testURI, String name, + String requestFile, IRI defaultGraphURI, Map inputNamedGraphs, IRI resultDefaultGraphURI, + Map resultNamedGraphs) { + super(displayName, testURI, name, requestFile, defaultGraphURI, inputNamedGraphs, resultDefaultGraphURI, + resultNamedGraphs); + List testToIgnore = new ArrayList<>(); + // @todo these tests are failing and should not, they are skipped so + // that we can be sure that we see when + // currently passing tests are not failing. Many of these tests are not + // so problematic since we do not support + // named graphs anyway + testToIgnore.add("DELETE INSERT 1b"); + testToIgnore.add("DELETE INSERT 1c"); + testToIgnore.add("CLEAR NAMED"); + testToIgnore.add("DROP NAMED"); + this.setIgnoredTests(testToIgnore); + } + + @Rule + public TemporaryFolder tempDir = TemporaryFolder.builder().assureDeletion().build(); + + @Override + protected Repository newRepository() throws Exception { + File nativeStore = tempDir.newFolder(); + File hdtStore = tempDir.newFolder(); + HDTOptions spec = HDTOptions.of(HDTOptionsKeys.DICTIONARY_TYPE_KEY, + HDTOptionsKeys.DICTIONARY_TYPE_VALUE_MULTI_OBJECTS, HDTOptionsKeys.BITMAPTRIPLES_INDEX_OTHERS, + EnumSet.of(TripleComponentOrder.SPO, TripleComponentOrder.OPS, TripleComponentOrder.PSO)); + Path fileName = Path.of(hdtStore.getAbsolutePath() + "/" + EndpointStoreTest.HDT_INDEX_NAME); + long size; + try (HDT hdt = Utility.createTempHdtIndex(tempDir, true, false, spec)) { + assert hdt != null; + size = hdt.getTriples().getNumberOfElements(); + hdt.saveToHDT(fileName, null); + } + + EndpointStore endpoint = new EndpointStore(hdtStore.getAbsolutePath() + "/", EndpointStoreTest.HDT_INDEX_NAME, + spec, nativeStore.getAbsolutePath() + "/", true) { + + @Override + public HDT loadIndex() throws IOException { + HDT idx = super.loadIndex(); + if (idx.getTriples().getNumberOfElements() == 0) { + return idx; + } + try { + Path fileOPS = BitmapTriplesIndexFile.getIndexPath(fileName, TripleComponentOrder.OPS); + Assert.assertTrue("can't find " + fileOPS, Files.exists(fileOPS)); + Path filePSO = BitmapTriplesIndexFile.getIndexPath(fileName, TripleComponentOrder.PSO); + Assert.assertTrue("can't find " + filePSO, Files.exists(filePSO)); + } catch (Throwable t) { + try (Stream l = Files.list(fileName.getParent())) { + l.forEach(System.err::println); + } catch (Exception e) { + t.addSuppressed(e); + } + throw t; + } + return idx; + } + }; + // endpoint.setThreshold(2); + + return new SailRepository(endpoint); + // return new DatasetRepository(new SailRepository(new + // NativeStore(tempDir.newFolder(), "spoc"))); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + } +} diff --git a/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/MergeRestartTest.java b/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/MergeRestartTest.java index cc020909..300d2311 100644 --- a/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/MergeRestartTest.java +++ b/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/MergeRestartTest.java @@ -221,7 +221,6 @@ private void mergeRestartTest1(MergeRunnableStopPoint stopPoint, File root, Clos executeTestAddHDT(countFile, endpointStore, 4, count); // show the bitmap and the file value - logger.debug("STEP1_TEST_BITMAP0o: {}", store.getDeleteBitMap().printInfo()); try (BitArrayDisk arr = new BitArrayDisk(0, hdtStore.getAbsolutePath() + "/triples-delete.arr")) { logger.debug("STEP1_TEST_BITMAP0n: {}", arr.printInfo()); } @@ -280,7 +279,6 @@ private void mergeRestartTest1(MergeRunnableStopPoint stopPoint, File root, Clos executeTestAddRDF(countFile, endpointStore, 4, count); executeTestAddHDT(countFile, endpointStore, 4, count); - logger.debug("count of deleted in hdt step2s: " + store.getDeleteBitMap().countOnes()); // test if the count is correct executeTestCount(countFile, endpointStore, store); ++step; @@ -304,7 +302,6 @@ private void mergeRestartTest1(MergeRunnableStopPoint stopPoint, File root, Clos executeTestAddRDF(countFile, endpointStore, 4, count); executeTestAddHDT(countFile, endpointStore, 4, count); - logger.debug("count of deleted in hdt step2e: " + store.getDeleteBitMap().countOnes()); // test if the count is correct executeTestCount(countFile, endpointStore, store); ++step; @@ -434,10 +431,6 @@ private void mergeRestartTest2(MergeRunnableStopPoint point, File root, Closer c MergeRunnableStopPoint.STEP2_START.debugWaitForEvent(); if (point.ordinal() <= MergeRunnableStopPoint.STEP2_START.ordinal()) { - logger.debug("test count 0"); - logger.debug("count of deleted in hdt: {}", store2.getDeleteBitMap().countOnes()); - if (store2.getTempDeleteBitMap() != null) - logger.debug("count of tmp del in hdt: {}", store2.getTempDeleteBitMap().countOnes()); // test if the count is correct executeTestCount(countFile, endpointStore2, store2); } @@ -445,10 +438,6 @@ private void mergeRestartTest2(MergeRunnableStopPoint point, File root, Closer c MergeRunnableStopPoint.STEP2_END.debugWaitForEvent(); if (point.ordinal() <= MergeRunnableStopPoint.STEP2_END.ordinal()) { - logger.debug("test count 1"); - logger.debug("count of deleted in hdt: {}", store2.getDeleteBitMap().countOnes()); - if (store2.getTempDeleteBitMap() != null) - logger.debug("count of tmp del in hdt: {}", store2.getTempDeleteBitMap().countOnes()); // test if the count is correct executeTestCount(countFile, endpointStore2, store2); } @@ -457,10 +446,6 @@ private void mergeRestartTest2(MergeRunnableStopPoint point, File root, Closer c // wait for the merge to complete MergeRunnable.debugWaitMerge(); - logger.debug("test count 2"); - logger.debug("count of deleted in hdt: {}", store2.getDeleteBitMap().countOnes()); - if (store2.getTempDeleteBitMap() != null) - logger.debug("count of tmp del in hdt: {}", store2.getTempDeleteBitMap().countOnes()); // test if the count is correct executeTestCount(countFile, endpointStore2, store2); } @@ -794,9 +779,6 @@ public static void printHDT(HDT hdt, EndpointStore store) { while (it.hasNext()) { logger.debug("- {}", it.next()); } - if (store != null) { - logger.debug("bitmap: {}", store.getDeleteBitMap().printInfo()); - } } /** diff --git a/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/Utility.java b/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/Utility.java index 7c11ab10..e851a1f9 100644 --- a/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/Utility.java +++ b/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/Utility.java @@ -90,7 +90,7 @@ public static HDT createTempHdtIndex(String fileName, boolean empty, boolean isB } catch (ParserException e) { throw new IOException("Can't generate hdt", e); } - return HDTManager.indexedHDT(hdt, null); + return HDTManager.indexedHDT(hdt, null, spec); } /**