GH-429 Add multi-indexes

the-qa-company · Nov 2, 2023 · 5232b01 · 5232b01
1 parent 4ede628
commit 5232b01
Show file tree

Hide file tree

Showing 40 changed files with 1,327 additions and 90 deletions.
diff --git a/...oint-core/src/main/java/com/the_qa_company/qendpoint/core/enums/TripleComponentOrder.java b/...oint-core/src/main/java/com/the_qa_company/qendpoint/core/enums/TripleComponentOrder.java
@@ -19,48 +19,97 @@
 
 package com.the_qa_company.qendpoint.core.enums;
 
+import java.util.Map;
+
 /**
  * Indicates the order of the triples
  */
 public enum TripleComponentOrder {
 	/**
 	 * Subject, predicate, object
 	 */
-	Unknown(null, null, null),
+	Unknown(null, null, null, 0),
 	/**
 	 * Subject, predicate, object
 	 */
-	SPO(TripleComponentRole.SUBJECT, TripleComponentRole.PREDICATE, TripleComponentRole.OBJECT),
+	SPO(TripleComponentRole.SUBJECT, TripleComponentRole.PREDICATE, TripleComponentRole.OBJECT, 1),
 	/**
 	 * Subject, object, predicate
 	 */
-	SOP(TripleComponentRole.SUBJECT, TripleComponentRole.OBJECT, TripleComponentRole.PREDICATE),
+	SOP(TripleComponentRole.SUBJECT, TripleComponentRole.OBJECT, TripleComponentRole.PREDICATE, 1 << 1),
 	/**
 	 * Predicate, subject, object
 	 */
-	PSO(TripleComponentRole.PREDICATE, TripleComponentRole.SUBJECT, TripleComponentRole.OBJECT),
+	PSO(TripleComponentRole.PREDICATE, TripleComponentRole.SUBJECT, TripleComponentRole.OBJECT, 1 << 2),
 	/**
 	 * Predicate, object, subject
 	 */
-	POS(TripleComponentRole.PREDICATE, TripleComponentRole.OBJECT, TripleComponentRole.SUBJECT),
+	POS(TripleComponentRole.PREDICATE, TripleComponentRole.OBJECT, TripleComponentRole.SUBJECT, 1 << 3),
 	/**
 	 * Object, subject, predicate
 	 */
-	OSP(TripleComponentRole.OBJECT, TripleComponentRole.SUBJECT, TripleComponentRole.PREDICATE),
+	OSP(TripleComponentRole.OBJECT, TripleComponentRole.SUBJECT, TripleComponentRole.PREDICATE, 1 << 4),
 	/**
 	 * Object, predicate, subject
 	 */
-	OPS(TripleComponentRole.OBJECT, TripleComponentRole.PREDICATE, TripleComponentRole.SUBJECT);
+	OPS(TripleComponentRole.OBJECT, TripleComponentRole.PREDICATE, TripleComponentRole.SUBJECT, 1 << 5);
+
+	public static final int ALL_MASK;
+
+	static {
+		int allMask = 0;
+		// add all the mask to the var
+		for (TripleComponentOrder order : values()) {
+			allMask |= order.mask;
+		}
+		ALL_MASK = allMask;
+	}
 
 	private final TripleComponentRole subjectMapping;
 	private final TripleComponentRole predicateMapping;
 	private final TripleComponentRole objectMapping;
+	public final int mask;
 
 	TripleComponentOrder(TripleComponentRole subjectMapping, TripleComponentRole predicateMapping,
-			TripleComponentRole objectMapping) {
+			TripleComponentRole objectMapping, int mask) {
 		this.subjectMapping = subjectMapping;
 		this.predicateMapping = predicateMapping;
 		this.objectMapping = objectMapping;
+		this.mask = mask;
+	}
+
+	/**
+	 * Search for an acceptable value in a map of orders
+	 *
+	 * @param flags flags to search the value
+	 * @param map   map
+	 * @param <T>   value type
+	 * @return find value, null for no matching value
+	 */
+	public static <T> T fetchBestForCfg(int flags, Map<? extends TripleComponentOrder, T> map) {
+		for (Map.Entry<? extends TripleComponentOrder, T> e : map.entrySet()) {
+			if ((e.getKey().mask & flags) != 0) {
+				return e.getValue();
+			}
+		}
+		return null;
+	}
+
+	/**
+	 * get an acceptable order for a order mask
+	 *
+	 * @param flags order mask
+	 * @return order, {@link #Unknown} if nothing was found
+	 */
+	public static TripleComponentOrder getAcceptableOrder(int flags) {
+		if (flags != 0) {
+			for (TripleComponentOrder v : values()) {
+				if ((v.mask & flags) == 0) {
+					return v;
+				}
+			}
+		}
+		return Unknown;
 	}
 
 	public TripleComponentRole getSubjectMapping() {

diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/HDTImpl.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/HDTImpl.java
@@ -33,6 +33,7 @@
 import com.the_qa_company.qendpoint.core.dictionary.impl.MultipleSectionDictionaryBig;
 import com.the_qa_company.qendpoint.core.dictionary.impl.MultipleSectionDictionaryCat;
 import com.the_qa_company.qendpoint.core.enums.ResultEstimationType;
+import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder;
 import com.the_qa_company.qendpoint.core.enums.TripleComponentRole;
 import com.the_qa_company.qendpoint.core.exceptions.IllegalFormatException;
 import com.the_qa_company.qendpoint.core.exceptions.NotFoundException;
@@ -83,6 +84,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Map;
 import java.util.zip.GZIPInputStream;
@@ -265,6 +267,18 @@ public void saveToHDT(String fileName, ProgressListener listener) throws IOExcep
 	@Override
 	public IteratorTripleString search(CharSequence subject, CharSequence predicate, CharSequence object)
 			throws NotFoundException {
+		return search(subject, predicate, object, TripleComponentOrder.ALL_MASK);
+	}
+
+	@Override
+	public IteratorTripleString search(CharSequence subject, CharSequence predicate, CharSequence object,
+			CharSequence graph) throws NotFoundException {
+		return search(subject, predicate, object, graph, TripleComponentOrder.ALL_MASK);
+	}
+
+	@Override
+	public IteratorTripleString search(CharSequence subject, CharSequence predicate, CharSequence object,
+			int searchOrderMask) throws NotFoundException {
 
 		if (isClosed) {
 			throw new IllegalStateException("Cannot search an already closed HDT");
@@ -314,29 +328,35 @@ public long estimatedNumResults() {
 				public long getLastTriplePosition() {
 					throw new NotImplementedException();
 				}
+
+				@Override
+				public TripleComponentOrder getOrder() {
+					return TripleComponentOrder.getAcceptableOrder(searchOrderMask);
+				}
 			};
 		}
 
 		CharSequence g = dictionary.supportGraphs() ? "" : null;
 
 		if (isMapped) {
 			try {
-				return new DictionaryTranslateIteratorBuffer(triples.search(triple), dictionary, subject, predicate,
-						object, g);
+				return new DictionaryTranslateIteratorBuffer(triples.search(triple, searchOrderMask), dictionary,
+						subject, predicate, object, g);
 			} catch (NullPointerException e) {
 				e.printStackTrace();
 				// FIXME: find why this can happen
-				return new DictionaryTranslateIterator(triples.search(triple), dictionary, subject, predicate, object,
-						g);
+				return new DictionaryTranslateIterator(triples.search(triple, searchOrderMask), dictionary, subject,
+						predicate, object, g);
 			}
 		} else {
-			return new DictionaryTranslateIterator(triples.search(triple), dictionary, subject, predicate, object, g);
+			return new DictionaryTranslateIterator(triples.search(triple, searchOrderMask), dictionary, subject,
+					predicate, object, g);
 		}
 	}
 
 	@Override
 	public IteratorTripleString search(CharSequence subject, CharSequence predicate, CharSequence object,
-			CharSequence graph) throws NotFoundException {
+			CharSequence graph, int searchOrderMask) throws NotFoundException {
 		if (isClosed) {
 			throw new IllegalStateException("Cannot search an already closed HDT");
 		}
@@ -386,22 +406,27 @@ public long estimatedNumResults() {
 				public long getLastTriplePosition() {
 					throw new NotImplementedException();
 				}
+
+				@Override
+				public TripleComponentOrder getOrder() {
+					return TripleComponentOrder.getAcceptableOrder(searchOrderMask);
+				}
 			};
 		}
 
 		if (isMapped) {
 			try {
-				return new DictionaryTranslateIteratorBuffer(triples.search(triple), dictionary, subject, predicate,
-						object, graph);
+				return new DictionaryTranslateIteratorBuffer(triples.search(triple, searchOrderMask), dictionary,
+						subject, predicate, object, graph);
 			} catch (NullPointerException e) {
 				e.printStackTrace();
 				// FIXME: find why this can happen
-				return new DictionaryTranslateIterator(triples.search(triple), dictionary, subject, predicate, object,
-						graph);
+				return new DictionaryTranslateIterator(triples.search(triple, searchOrderMask), dictionary, subject,
+						predicate, object, graph);
 			}
 		} else {
-			return new DictionaryTranslateIterator(triples.search(triple), dictionary, subject, predicate, object,
-					graph);
+			return new DictionaryTranslateIterator(triples.search(triple, searchOrderMask), dictionary, subject,
+					predicate, object, graph);
 		}
 	}
 
@@ -454,6 +479,13 @@ public void loadOrCreateIndex(ProgressListener listener, HDTOptions spec) throws
 			// We need no index.
 			return;
 		}
+		triples.mapGenOtherIndexes(Path.of(String.valueOf(hdtFileName)), spec, listener);
+
+		// disable the FOQ generation if asked
+		if (spec.getBoolean(HDTOptionsKeys.BITMAPTRIPLES_INDEX_NO_FOQ, false)) {
+			return;
+		}
+
 		ControlInfo ci = new ControlInformation();
 		String indexName = hdtFileName + HDTVersion.get_index_suffix("-");
 		indexName = indexName.replaceAll("\\.hdt\\.gz", "hdt");

diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/WriteHDTImpl.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/WriteHDTImpl.java
@@ -2,6 +2,7 @@
 
 import com.the_qa_company.qendpoint.core.dictionary.DictionaryFactory;
 import com.the_qa_company.qendpoint.core.dictionary.DictionaryPrivate;
+import com.the_qa_company.qendpoint.core.exceptions.NotFoundException;
 import com.the_qa_company.qendpoint.core.exceptions.NotImplementedException;
 import com.the_qa_company.qendpoint.core.header.HeaderFactory;
 import com.the_qa_company.qendpoint.core.header.HeaderPrivate;
@@ -122,4 +123,11 @@ public void close() throws IOException {
 	public IteratorTripleString search(CharSequence subject, CharSequence predicate, CharSequence object) {
 		throw new NotImplementedException();
 	}
+
+	@Override
+	public IteratorTripleString search(CharSequence subject, CharSequence predicate, CharSequence object,
+			int searchOrderMask) throws NotFoundException {
+		throw new NotImplementedException();
+	}
+
 }
diff --git a/...ore/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/diskimport/MapOnCallHDT.java b/...ore/src/main/java/com/the_qa_company/qendpoint/core/hdt/impl/diskimport/MapOnCallHDT.java
@@ -9,6 +9,7 @@
 import com.the_qa_company.qendpoint.core.listener.ProgressListener;
 import com.the_qa_company.qendpoint.core.options.HDTOptions;
 import com.the_qa_company.qendpoint.core.triples.IteratorTripleString;
+import com.the_qa_company.qendpoint.core.triples.TripleString;
 import com.the_qa_company.qendpoint.core.triples.Triples;
 import com.the_qa_company.qendpoint.core.util.io.CloseSuppressPath;
 import com.the_qa_company.qendpoint.core.util.io.IOUtil;
@@ -20,6 +21,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.util.Iterator;
 
 /**
  * HDT implementation delaying the map method to avoid mapping into memory a
@@ -107,6 +109,43 @@ public IteratorTripleString search(CharSequence subject, CharSequence predicate,
 		return mapOrGetHDT().search(subject, predicate, object, graph);
 	}
 
+	@Override
+	public IteratorTripleString search(TripleString triple) throws NotFoundException {
+		return mapOrGetHDT().search(triple);
+	}
+
+	@Override
+	public IteratorTripleString searchAll() throws NotFoundException {
+		return mapOrGetHDT().searchAll();
+	}
+
+	@Override
+	public IteratorTripleString search(CharSequence subject, CharSequence predicate, CharSequence object,
+			int searchOrderMask) throws NotFoundException {
+		return mapOrGetHDT().search(subject, predicate, object, searchOrderMask);
+	}
+
+	@Override
+	public IteratorTripleString search(CharSequence subject, CharSequence predicate, CharSequence object,
+			CharSequence graph, int searchOrderMask) throws NotFoundException {
+		return mapOrGetHDT().search(subject, predicate, object, graph, searchOrderMask);
+	}
+
+	@Override
+	public IteratorTripleString search(TripleString triple, int searchOrderMask) throws NotFoundException {
+		return mapOrGetHDT().search(triple, searchOrderMask);
+	}
+
+	@Override
+	public IteratorTripleString searchAll(int searchOrderMask) throws NotFoundException {
+		return mapOrGetHDT().searchAll(searchOrderMask);
+	}
+
+	@Override
+	public Iterator<TripleString> iterator() {
+		return mapOrGetHDT().iterator();
+	}
+
 	@Override
 	public void loadFromHDT(InputStream input, ProgressListener listener) throws IOException {
 		((HDTPrivate) mapOrGetHDT()).loadFromHDT(input, listener);

diff --git a/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/header/PlainHeader.java b/qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/header/PlainHeader.java
@@ -27,6 +27,7 @@
 import java.util.List;
 
 import com.the_qa_company.qendpoint.core.exceptions.IllegalFormatException;
+import com.the_qa_company.qendpoint.core.exceptions.NotFoundException;
 import com.the_qa_company.qendpoint.core.exceptions.ParserException;
 import com.the_qa_company.qendpoint.core.listener.ProgressListener;
 import com.the_qa_company.qendpoint.core.options.ControlInfo;
@@ -188,6 +189,18 @@ public IteratorTripleString search(CharSequence subject, CharSequence predicate,
 		return new PlainHeaderIterator(this, pattern);
 	}
 
+	@Override
+	public IteratorTripleString search(CharSequence subject, CharSequence predicate, CharSequence object,
+			int searchOrderMask) throws NotFoundException {
+		return search(subject, predicate, object);
+	}
+
+	@Override
+	public IteratorTripleString search(CharSequence subject, CharSequence predicate, CharSequence object,
+			CharSequence graph, int searchOrderMask) throws NotFoundException {
+		return search(subject, predicate, object, graph);
+	}
+
 	@Override
 	public void processTriple(TripleString triple, long pos) {
 		triples.add(new TripleString(triple));

diff --git a/...oint-core/src/main/java/com/the_qa_company/qendpoint/core/header/PlainHeaderIterator.java b/...oint-core/src/main/java/com/the_qa_company/qendpoint/core/header/PlainHeaderIterator.java
@@ -19,6 +19,7 @@
 
 package com.the_qa_company.qendpoint.core.header;
 
+import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder;
 import com.the_qa_company.qendpoint.core.triples.IteratorTripleString;
 import com.the_qa_company.qendpoint.core.triples.TripleString;
 import com.the_qa_company.qendpoint.core.enums.ResultEstimationType;
@@ -111,4 +112,8 @@ public long getLastTriplePosition() {
 		throw new UnsupportedOperationException();
 	}
 
+	@Override
+	public TripleComponentOrder getOrder() {
+		return TripleComponentOrder.Unknown;
+	}
 }
diff --git a/...src/main/java/com/the_qa_company/qendpoint/core/iterator/DictionaryTranslateIterator.java b/...src/main/java/com/the_qa_company/qendpoint/core/iterator/DictionaryTranslateIterator.java
@@ -21,6 +21,7 @@
 
 import com.the_qa_company.qendpoint.core.dictionary.Dictionary;
 import com.the_qa_company.qendpoint.core.enums.ResultEstimationType;
+import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder;
 import com.the_qa_company.qendpoint.core.enums.TripleComponentRole;
 import com.the_qa_company.qendpoint.core.quad.QuadString;
 import com.the_qa_company.qendpoint.core.triples.IteratorTripleID;
@@ -161,4 +162,9 @@ public long getLastTriplePosition() {
 		return iterator.getLastTriplePosition();
 	}
 
+	@Override
+	public TripleComponentOrder getOrder() {
+		return iterator.getOrder();
+	}
+
 }
diff --git a/...in/java/com/the_qa_company/qendpoint/core/iterator/DictionaryTranslateIteratorBuffer.java b/...in/java/com/the_qa_company/qendpoint/core/iterator/DictionaryTranslateIteratorBuffer.java
@@ -21,6 +21,7 @@
 import com.the_qa_company.qendpoint.core.dictionary.DictionaryPrivate;
 import com.the_qa_company.qendpoint.core.dictionary.impl.OptimizedExtractor;
 import com.the_qa_company.qendpoint.core.enums.ResultEstimationType;
+import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder;
 import com.the_qa_company.qendpoint.core.enums.TripleComponentRole;
 import com.the_qa_company.qendpoint.core.quad.QuadString;
 import com.the_qa_company.qendpoint.core.triples.IteratorTripleString;
@@ -278,6 +279,11 @@ public long getLastTriplePosition() {
 		return lastPosition.compute();
 	}
 
+	@Override
+	public TripleComponentOrder getOrder() {
+		return iterator.getOrder();
+	}
+
 	public static void setBlockSize(int size) {
 		DEFAULT_BLOCK_SIZE = size;
 	}