Merge pull request #445 from the-qa-company/dev-fix-co-indexes

Add signature to idx files
the-qa-company · Jan 24, 2024 · 351595e · 351595e
2 parents a697f7b + e33d93f
commit 351595e
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 6 deletions.
diff --git a/...oint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java b/...oint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java
@@ -1339,7 +1339,7 @@ public void syncOtherIndexes(Path fileLocation, HDTOptions spec, ProgressListene
 			try (FileChannel channel = FileChannel.open(subIndexPath, StandardOpenOption.READ)) {
 				// load from the path...
 
-				BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel);
+				BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel, this);
 				BitmapTriplesIndex old = indexes.put(order, idx);
 				indexesMask |= idx.getOrder().mask;
 				if (old != null) {
@@ -1356,7 +1356,7 @@ public void syncOtherIndexes(Path fileLocation, HDTOptions spec, ProgressListene
 				BitmapTriplesIndexFile.generateIndex(this, subIndexPath, order, spec, mListener);
 				try (FileChannel channel = FileChannel.open(subIndexPath, StandardOpenOption.READ)) {
 					// load from the path...
-					BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel);
+					BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel, this);
 					BitmapTriplesIndex old = indexes.put(order, idx);
 					indexesMask |= order.mask;
 					if (old != null) {

diff --git a/.../src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java b/.../src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java
@@ -33,6 +33,7 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InterruptedIOException;
+import java.nio.ByteOrder;
 import java.nio.channels.Channels;
 import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
@@ -59,30 +60,47 @@ public static Path getIndexPath(Path hdt, TripleComponentOrder order) {
 		return hdt.resolveSibling(hdt.getFileName() + "." + order.name().toLowerCase() + ".idx");
 	}
 
-	public static final byte[] MAGIC = "$HDTIDX0".getBytes(StandardCharsets.US_ASCII);
+	/**
+	 * Compute triples signature
+	 * @param triples triples
+	 * @return signature
+	 */
+	public static long signature(BitmapTriples triples) {
+		return 0x484454802020L ^ triples.getNumberOfElements();
+	}
+
+	public static final byte[] MAGIC = "$HDTIDX1".getBytes(StandardCharsets.US_ASCII);
 
 	/**
 	 * Map a file from a file
 	 *
 	 * @param file    file
 	 * @param channel channel
+	 * @param triples    triples
 	 * @return index
 	 * @throws IOException io
 	 */
-	public static BitmapTriplesIndex map(Path file, FileChannel channel) throws IOException {
+	public static BitmapTriplesIndex map(Path file, FileChannel channel, BitmapTriples triples) throws IOException {
 		try (CloseMappedByteBuffer header = IOUtil.mapChannel(file, channel, FileChannel.MapMode.READ_ONLY, 0,
-				MAGIC.length)) {
+				MAGIC.length + 8)) {
 			byte[] magicRead = new byte[MAGIC.length];
 
 			header.get(magicRead);
 
 			if (!Arrays.equals(magicRead, MAGIC)) {
 				throw new IOException(format("Can't read %s magic", file));
 			}
+
+			long signature = header.order(ByteOrder.LITTLE_ENDIAN).getLong(magicRead.length);
+
+			long currentSignature = signature(triples);
+			if (signature != currentSignature) {
+				throw new IOException(format("Wrong signature for file 0x%x != 0x%x", signature, currentSignature));
+			}
 		}
 
 		CountInputStream stream = new CountInputStream(new BufferedInputStream(Channels.newInputStream(channel)));
-		stream.skipNBytes(MAGIC.length);
+		stream.skipNBytes(MAGIC.length + 8);
 
 		String orderCfg = IOUtil.readSizedString(stream, ProgressListener.ignore());
 
@@ -268,6 +286,7 @@ public static void generateIndex(BitmapTriples triples, Path destination, Triple
 				// saving the index
 				try (BufferedOutputStream output = new BufferedOutputStream(Files.newOutputStream(destination))) {
 					output.write(MAGIC);
+					IOUtil.writeLong(output, signature(triples));
 
 					IOUtil.writeSizedString(output, order.name(), listener);