GH-477 Reimplement #470

wip (#470) * wip [skip ci] * wip * test sync * test zqzq * test zqzqsdfghjk * yolo * fix qep mapping, reduce logs for diff * wip [skip ci] * tests
the-qa-company · Sep 10, 2024 · 3f66d65 · 3f66d65
1 parent 6b2cfab
commit 3f66d65
Show file tree

Hide file tree

Showing 29 changed files with 1,184 additions and 469 deletions.
diff --git a/...int-backend/src/main/java/com/the_qa_company/qendpoint/controller/EndpointController.java b/...int-backend/src/main/java/com/the_qa_company/qendpoint/controller/EndpointController.java
@@ -190,8 +190,15 @@ public ResponseEntity<Sparql.IsDumpingResult> isDumping() {
 	}
 
 	@GetMapping("/reindex")
-	public ResponseEntity<Sparql.LuceneIndexRequestResult> reindex() throws Exception {
-		return ResponseEntity.status(HttpStatus.OK).body(sparql.reindexLucene());
+	public ResponseEntity<Sparql.LuceneIndexRequestResult> reindex(
+			@RequestParam(value = "index", defaultValue = "all") final String index) throws Exception {
+		return ResponseEntity.status(HttpStatus.OK)
+				.body(index == null || index.equals("all") ? sparql.reindexLucene() : sparql.reindexLucene(index));
+	}
+
+	@GetMapping("/lucenes")
+	public ResponseEntity<Sparql.LuceneIndexListResult> lucenes() {
+		return ResponseEntity.status(HttpStatus.OK).body(sparql.lucenes());
 	}
 
 	@GetMapping("/has_index")

diff --git a/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java b/qendpoint-backend/src/main/java/com/the_qa_company/qendpoint/controller/Sparql.java
@@ -1,6 +1,5 @@
 package com.the_qa_company.qendpoint.controller;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.the_qa_company.qendpoint.client.QEndpointClient;
 import com.the_qa_company.qendpoint.client.SplitStream;
 import com.the_qa_company.qendpoint.compiler.CompiledSail;
@@ -12,7 +11,6 @@
 import com.the_qa_company.qendpoint.store.EndpointStoreUtils;
 import com.the_qa_company.qendpoint.utils.FileUtils;
 import com.the_qa_company.qendpoint.utils.RDFStreamUtils;
-import org.eclipse.rdf4j.model.Namespace;
 import org.eclipse.rdf4j.model.Statement;
 import org.eclipse.rdf4j.model.util.Values;
 import org.eclipse.rdf4j.repository.RepositoryConnection;
@@ -29,6 +27,7 @@
 import com.the_qa_company.qendpoint.core.util.StopWatch;
 import com.the_qa_company.qendpoint.core.util.io.CloseSuppressPath;
 import com.the_qa_company.qendpoint.core.util.io.IOUtil;
+import org.eclipse.rdf4j.sail.lucene.LuceneSail;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
@@ -53,6 +52,7 @@
 import java.time.format.DateTimeFormatter;
 import java.util.Calendar;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.function.Consumer;
@@ -62,138 +62,19 @@
 public class Sparql {
 	private static boolean redirectDone;
 
-	public static class NamespaceData {
-		private String prefix;
-		private String name;
+	public record MergeRequestResult(boolean completed) {}
 
-		public NamespaceData(String prefix, String name) {
-			this.prefix = prefix;
-			this.name = name;
-		}
+	public record LuceneIndexRequestResult(boolean completed) {}
 
-		public NamespaceData() {
-		}
+	public record LuceneIndexListResult(List<String> indexes) {}
 
-		public NamespaceData(Namespace ns) {
-			this(ns.getPrefix(), ns.getName());
-		}
+	public record HasLuceneIndexResult(boolean hasLuceneIndex) {}
 
-		public String getPrefix() {
-			return prefix;
-		}
+	public record IsMergingResult(boolean merging) {}
 
-		public String getName() {
-			return name;
-		}
-
-		public void setPrefix(String prefix) {
-			this.prefix = prefix;
-		}
-
-		public void setName(String name) {
-			this.name = name;
-		}
-
-		@JsonIgnore
-		public Namespace asRDF4J() {
-			return Values.namespace(getPrefix(), getName());
-		}
-	}
-
-	public static class NamespaceDataResult {
-		private Map<String, String> prefixes;
-
-		public NamespaceDataResult(Map<String, String> prefixes) {
-			this.prefixes = prefixes;
-		}
-
-		public NamespaceDataResult() {
-		}
+	public record IsDumpingResult(boolean dumping) {}
 
-		public Map<String, String> getPrefixes() {
-			return prefixes;
-		}
-
-		public void setPrefixes(Map<String, String> prefixes) {
-			this.prefixes = prefixes;
-		}
-	}
-
-	public static class MergeRequestResult {
-		private final boolean completed;
-
-		public MergeRequestResult(boolean completed) {
-			this.completed = completed;
-		}
-
-		public boolean isCompleted() {
-			return completed;
-		}
-	}
-
-	public static class LuceneIndexRequestResult {
-		private final boolean completed;
-
-		public LuceneIndexRequestResult(boolean completed) {
-			this.completed = completed;
-		}
-
-		public boolean isCompleted() {
-			return completed;
-		}
-	}
-
-	public static class HasLuceneIndexResult {
-		private final boolean hasLuceneIndex;
-
-		public HasLuceneIndexResult(boolean hasLuceneIndex) {
-			this.hasLuceneIndex = hasLuceneIndex;
-		}
-
-		public boolean isHasLuceneIndex() {
-			return hasLuceneIndex;
-		}
-	}
-
-	public static class IsMergingResult {
-		private final boolean merging;
-
-		public IsMergingResult(boolean merging) {
-			this.merging = merging;
-		}
-
-		public boolean isMerging() {
-			return merging;
-		}
-	}
-
-	public static class IsDumpingResult {
-		private final boolean dumping;
-
-		public IsDumpingResult(boolean dumping) {
-			this.dumping = dumping;
-		}
-
-		public boolean isDumping() {
-			return dumping;
-		}
-	}
-
-	public static class LoadFileResult {
-		private boolean loaded;
-
-		public LoadFileResult(boolean loaded) {
-			this.loaded = loaded;
-		}
-
-		public boolean isLoaded() {
-			return loaded;
-		}
-
-		public void setLoaded(boolean loaded) {
-			this.loaded = loaded;
-		}
-	}
+	public record LoadFileResult(boolean loaded) {}
 
 	private static final Logger logger = LoggerFactory.getLogger(Sparql.class);
 	private static final SimpleDateFormat FORMAT = new SimpleDateFormat("yyyyMMdd-HHmmss");
@@ -526,6 +407,17 @@ public LuceneIndexRequestResult reindexLucene() throws Exception {
 		return new LuceneIndexRequestResult(true);
 	}
 
+	public LuceneIndexRequestResult reindexLucene(String index) throws Exception {
+		initializeEndpointStore(true);
+		sparqlRepository.reindexLuceneSail(index);
+		return new LuceneIndexRequestResult(true);
+	}
+
+	public LuceneIndexListResult lucenes() {
+		return new LuceneIndexListResult(
+				sparqlRepository.getLuceneSails().stream().map(lc -> lc.getParameter(LuceneSail.INDEX_ID)).toList());
+	}
+
 	/**
 	 * @return if the sail has a least one lucene sail connected to it
 	 */

diff --git a/...t-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/kcat/KCatImpl.java b/...t-core/src/main/java/com/the_qa_company/qendpoint/core/dictionary/impl/kcat/KCatImpl.java
@@ -266,9 +266,12 @@ private KCatImpl(List<?> hdtFileNames, List<? extends Bitmap> deleteBitmaps, HDT
 					while (searchAll.hasNext()) {
 						TripleID tripleID = searchAll.next();
 
-						iListener.notifyProgress((float) (c++ * 10000 / numberOfElements) / 100f,
-								"building diff bitmaps " + c + "/" + numberOfElements + " (hdt " + index + "/"
-										+ hdts.length + ")");
+						if (c % 10000 == 0) {
+							iListener.notifyProgress((float) (c * 10000 / numberOfElements) / 100f,
+									"building diff bitmaps " + c + "/" + numberOfElements + " (hdt " + index + "/"
+											+ hdts.length + ")");
+						}
+						c++;
 
 						long g = quad ? (tripleID.getGraph() - 1) : 0;