diff --git a/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java b/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java index 7b25ce0c..d623192f 100644 --- a/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java +++ b/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java @@ -456,7 +456,10 @@ public LoadFileResult loadFile(InputStream input, String filename) throws IOExce Files.createDirectories(Paths.get(locationHdt)); Files.deleteIfExists(Paths.get(hdtOutput)); - Files.deleteIfExists(Paths.get(EndpointFiles.getHDTIndexV11(locationHdt, hdtIndexName))); + + for (String indexFile : EndpointFiles.getHDTIndexNames(locationHdt, hdtIndexName)) { + Files.deleteIfExists(Path.of(indexFile)); + } if (sparqlRepository.getOptions().getStorageMode().equals(SailCompilerSchema.ENDPOINTSTORE_STORAGE)) { shutdown(); diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointFiles.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointFiles.java index 25cff0d5..55b80468 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointFiles.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointFiles.java @@ -3,9 +3,12 @@ import com.the_qa_company.qendpoint.compiler.ParsedStringValue; import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder; import com.the_qa_company.qendpoint.core.hdt.HDTVersion; +import com.the_qa_company.qendpoint.core.triples.impl.BitmapTriplesIndexFile; import java.io.File; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; /** * store the files used by the endpoint store @@ -34,6 +37,42 @@ public static String getHDTIndexV11(String locationHdt, String hdtIndexName) { return locationHdt + hdtIndexName + HDTVersion.get_index_suffix("-"); } + public static List getHDTIndexNames(String locationHdt, String hdtIndexName) { + Path path = Path.of(locationHdt + hdtIndexName); + + List list = new ArrayList<>(); + + Path hdtPath = Path.of(getHDTIndex(locationHdt, hdtIndexName)); + + list.add(getHDTIndexV11(locationHdt, hdtIndexName)); + + for (TripleComponentOrder order : TripleComponentOrder.values()) { + if (order == TripleComponentOrder.Unknown) + continue; + list.add(BitmapTriplesIndexFile.getIndexPath(hdtPath, order).toAbsolutePath().toString()); + } + + return list; + } + + public static List getHDTIndexNamesPath(String locationHdt, String hdtIndexName) { + Path path = Path.of(locationHdt + hdtIndexName); + + List list = new ArrayList<>(); + + Path hdtPath = Path.of(getHDTIndex(locationHdt, hdtIndexName)); + + list.add(Path.of(getHDTIndexV11(locationHdt, hdtIndexName))); + + for (TripleComponentOrder order : TripleComponentOrder.values()) { + if (order == TripleComponentOrder.Unknown) + continue; + list.add(BitmapTriplesIndexFile.getIndexPath(hdtPath, order)); + } + + return list; + } + // basic locations private final String locationNative, locationHdt, hdtIndexName; private final Path locationNativePath, locationHdtPath; @@ -184,6 +223,18 @@ public String getHDTIndexV11() { return getHDTIndexV11(locationHdt, hdtIndexName); } + public List getHDTIndexNames() { + return getHDTIndexNames(locationHdt, hdtIndexName); + } + + public List getHDTNewIndexNames() { + return getHDTIndexNames(locationHdt, hdtIndexName + ".new.hdt"); + } + + public List getHDTNewIndexNamesPath() { + return getHDTIndexNamesPath(locationHdt, hdtIndexName + ".new.hdt"); + } + /** * @return the HDT file */ @@ -191,13 +242,6 @@ public Path getHDTIndexPath() { return Path.of(getHDTIndex()); } - /** - * @return the HDT file with HDT version - */ - public Path getHDTIndexV11Path() { - return Path.of(getHDTIndexV11()); - } - /** * @return the new HDT file */ diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreDump.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreDump.java index 2ffb65a1..40cfe4a4 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreDump.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/EndpointStoreDump.java @@ -1,10 +1,9 @@ package com.the_qa_company.qendpoint.store; -import com.the_qa_company.qendpoint.core.hdt.HDTVersion; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; /** * information on how a {@link EndpointStore} should be dumped @@ -36,9 +35,31 @@ default void beforeMerge(EndpointStore store) throws IOException { default void afterMerge(EndpointStore store, Path mergedDataset) throws IOException { } + /** + * call after the indexing + * + * @param store store + * @param mergedDatasetIndex v11 index + * @throws IOException any + * @deprecated use {@link #afterIndexing(EndpointStore, List)} instead + */ + @Deprecated default void afterIndexing(EndpointStore store, Path mergedDatasetIndex) throws IOException { } + /** + * call after the indexing + * + * @param store store + * @param mergedDatasetIndexes indexes + * @throws IOException any + */ + default void afterIndexing(EndpointStore store, List mergedDatasetIndexes) throws IOException { + if (!mergedDatasetIndexes.isEmpty()) { + afterIndexing(store, mergedDatasetIndexes.get(0)); + } + } + class EndpointStoreDumpDataset implements EndpointStoreDump { protected final Path outputLocation; @@ -53,11 +74,25 @@ public void afterMerge(EndpointStore store, Path mergedDataset) throws IOExcepti Files.copy(mergedDataset, outputLocation.resolve("store.hdt")); } + static String replaceHDTFilename(Path path, String newName) { + String filename = path.getFileName().toString(); + + int idx = filename.lastIndexOf(".hdt"); + + if (idx == -1) + throw new IllegalArgumentException("Not a HDT file"); + + return newName + filename.substring(idx); + } + @Override - public void afterIndexing(EndpointStore store, Path mergedDatasetIndex) throws IOException { + public void afterIndexing(EndpointStore store, List mergedDatasetIndexes) throws IOException { Files.createDirectories(outputLocation.getParent()); // store the dataset - Files.copy(mergedDatasetIndex, outputLocation.resolve("store.hdt" + HDTVersion.get_index_suffix("-"))); + for (Path path : mergedDatasetIndexes) { + Files.copy(path, outputLocation.resolve(replaceHDTFilename(path, "store"))); + + } } } } diff --git a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/MergeRunnable.java b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/MergeRunnable.java index 005b75ea..2db6ac92 100644 --- a/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/MergeRunnable.java +++ b/qendpoint-store/src/main/java/com/the_qa_company/qendpoint/store/MergeRunnable.java @@ -197,6 +197,14 @@ private static boolean exists(String file) { return Files.exists(Path.of(file)); } + private static boolean existsAny(List files) { + for (String file : files) { + if (Files.exists(Path.of(file))) + return true; + } + return false; + } + /** * test if the file "file + {@link #OLD_EXT}" exists * @@ -207,6 +215,21 @@ private static boolean existsOld(String file) { return exists(file + OLD_EXT); } + /** + * test if the file "file + {@link #OLD_EXT}" exists for file in files + * + * @param files the files + * @return true if the file exists, false otherwise + */ + private static boolean existsOldAny(List files) { + for (String file : files) { + if (exists(file + OLD_EXT)) { + return true; + } + } + return false; + } + /** * rename file to "file + {@link #OLD_EXT}" * @@ -225,6 +248,14 @@ private static void renameFromOld(String file) { rename(file + OLD_EXT, file); } + private static void renameFromOld(List files) { + for (String file : files) { + if (Files.exists(Path.of(file + OLD_EXT))) { + rename(file + OLD_EXT, file); + } + } + } + /** * rename a file to another * @@ -241,6 +272,29 @@ private static void rename(String oldFile, String newFile) { } } + /** + * rename a file to another list + * + * @param oldFile the current name + * @param newFile the new name + */ + private static void rename(List oldFile, List newFile) { + if (oldFile.size() != newFile.size()) + throw new RuntimeException("Bad list count: " + oldFile.size() + "/" + newFile.size()); + try { + for (int i = 0; i < oldFile.size(); i++) { + Path old = Path.of(oldFile.get(i)); + if (Files.exists(old)) { + Files.move(old, Path.of(newFile.get(i)), StandardCopyOption.REPLACE_EXISTING); + } + } + } catch (IOException e) { + logger.warn("Can't rename the file {} into {} ({})", oldFile, newFile, e.getClass().getName()); + if (MergeRunnableStopPoint.debug) + throw new RuntimeException(e); + } + } + private static final Logger logger = LoggerFactory.getLogger(MergeRunnable.class); public class MergeThread extends Thread { @@ -686,7 +740,7 @@ private Step3SubStep getStep3SubStep() { boolean existsOldTripleDeleteTempArr = endpoint.getValidOrders().stream() .anyMatch(order -> existsOld(endpointFiles.getTripleDeleteTempArr(order))); - if (existsOldTripleDeleteTempArr && !exists(endpointFiles.getHDTNewIndexV11())) { + if (existsOldTripleDeleteTempArr && !existsAny(endpointFiles.getHDTNewIndexNames())) { // after rename(endpointFiles.getHDTNewIndexV11(), // endpointFiles.getHDTIndexV11()); return Step3SubStep.AFTER_INDEX_V11_RENAME; @@ -696,7 +750,7 @@ private Step3SubStep getStep3SubStep() { // endpointFiles.getHDTIndex()); return Step3SubStep.AFTER_INDEX_RENAME; } - if (existsOld(endpointFiles.getHDTIndexV11())) { + if (existsOldAny(endpointFiles.getHDTIndexNames())) { // after renameToOld(endpointFiles.getHDTIndexV11()); return Step3SubStep.AFTER_INDEX_V11_OLD_RENAME; } @@ -725,11 +779,11 @@ private void preloadStep3() { switch (step3SubStep) { case AFTER_INDEX_V11_RENAME: - rename(endpointFiles.getHDTIndexV11(), endpointFiles.getHDTNewIndexV11()); + rename(endpointFiles.getHDTIndexNames(), endpointFiles.getHDTNewIndexNames()); case AFTER_INDEX_RENAME: rename(endpointFiles.getHDTIndex(), endpointFiles.getHDTNewIndex()); case AFTER_INDEX_V11_OLD_RENAME: - renameFromOld(endpointFiles.getHDTIndexV11()); + renameFromOld(endpointFiles.getHDTIndexNames()); case AFTER_INDEX_OLD_RENAME: renameFromOld(endpointFiles.getHDTIndex()); case AFTER_TRIPLEDEL_TMP_OLD_RENAME: @@ -773,7 +827,7 @@ private synchronized void step3(boolean restarting, Lock lock, EndpointStoreDump boolean graph; try (HDT newHdt = HDTManager.mapIndexedHDT(endpointFiles.getHDTNewIndex(), endpoint.getHDTSpec(), null)) { if (dumpInfo != null) { - dumpInfo.afterIndexing(endpoint, Path.of(endpointFiles.getHDTNewIndexV11())); + dumpInfo.afterIndexing(endpoint, endpointFiles.getHDTNewIndexNamesPath()); endpoint.setDumping(endpoint.getDumpRef().get() != null); } graph = newHdt.getDictionary().supportGraphs(); @@ -795,12 +849,15 @@ private synchronized void step3(boolean restarting, Lock lock, EndpointStoreDump this.endpoint.resetDeleteArray(newHdt); } - Path hdtIndexV11 = Path.of(endpointFiles.getHDTIndexV11()); - // if the index.hdt.index.v1-1 doesn't exist, the hdt is empty, so we - // create a mock index file - // (ignored by RDF-HDT) - if (!Files.exists(hdtIndexV11)) { - Files.writeString(hdtIndexV11, ""); + if (!endpoint.getHDTSpec().getBoolean(HDTOptionsKeys.BITMAPTRIPLES_INDEX_NO_FOQ, false)) { + Path hdtIndexV11 = Path.of(endpointFiles.getHDTIndexV11()); + // if the index.hdt.index.v1-1 doesn't exist, the hdt is empty, so + // we + // create a mock index file + // (ignored by RDF-HDT) + if (!Files.exists(hdtIndexV11)) { + Files.writeString(hdtIndexV11, ""); + } } // rename new hdt to old hdt name so that they are replaces @@ -817,7 +874,7 @@ private synchronized void step3(boolean restarting, Lock lock, EndpointStoreDump rename(endpointFiles.getHDTNewIndex(), endpointFiles.getHDTIndex()); debugStepPoint(MergeRunnableStopPoint.STEP3_FILES_MID2); // AFTER_INDEX_RENAME - rename(endpointFiles.getHDTNewIndexV11(), endpointFiles.getHDTIndexV11()); + rename(endpointFiles.getHDTNewIndexNames(), endpointFiles.getHDTIndexNames()); // AFTER_INDEX_V11_RENAME HDT tempHdt = endpoint.loadIndex(); diff --git a/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/EndpointStoreDumpTest.java b/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/EndpointStoreDumpTest.java new file mode 100644 index 00000000..d80958b3 --- /dev/null +++ b/qendpoint-store/src/test/java/com/the_qa_company/qendpoint/store/EndpointStoreDumpTest.java @@ -0,0 +1,22 @@ +package com.the_qa_company.qendpoint.store; + +import org.junit.Test; + +import java.nio.file.Path; + +import static org.junit.Assert.*; + +public class EndpointStoreDumpTest { + + @Test + public void nameTest() { + assertEquals("aaaa.hdt", + EndpointStoreDump.EndpointStoreDumpDataset.replaceHDTFilename(Path.of("test.hdt"), "aaaa")); + assertEquals("bbbb.hdt", + EndpointStoreDump.EndpointStoreDumpDataset.replaceHDTFilename(Path.of("test.hdt"), "bbbb")); + assertEquals("bbbb.hdt.idx", + EndpointStoreDump.EndpointStoreDumpDataset.replaceHDTFilename(Path.of("test.hdt.idx"), "bbbb")); + assertEquals("bbbb.hdt", + EndpointStoreDump.EndpointStoreDumpDataset.replaceHDTFilename(Path.of("test.idx.hdt"), "bbbb")); + } +}