validityDates = new ArrayList<>();
@JsonBackReference
+ @EqualsAndHashCode.Exclude
private Concept concept;
@JsonIgnore
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java
index 85cf6322b8..edbf99b1c3 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java
@@ -224,6 +224,7 @@ public static FrontendTable createTable(Connector con) {
if(con.getValidityDates().size() > 1) {
result.setDateColumn(
new FrontendValidityDate(
+ con.getValidityDatesDescription(),
null,
con
.getValidityDates()
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java
index acae609303..1047c74a4c 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java
@@ -6,7 +6,10 @@
import com.bakdata.conquery.apiv1.frontend.FrontendValue;
import com.bakdata.conquery.io.storage.NamespaceStorage;
-import com.bakdata.conquery.models.config.SearchConfig;
+import com.bakdata.conquery.models.config.IndexConfig;
+import com.bakdata.conquery.models.datasets.Dataset;
+import com.bakdata.conquery.models.identifiable.Identifiable;
+import com.bakdata.conquery.models.identifiable.ids.Id;
import com.bakdata.conquery.models.query.FilterSearch;
import com.bakdata.conquery.util.search.TrieSearch;
import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -16,18 +19,22 @@
*
* Searchable classes describe how a search should be constructed, and provide the values with getSearchValues.
*/
-public interface Searchable {
+public interface Searchable>> extends Identifiable {
+
+ public Dataset getDataset();
+
/**
* All available {@link FrontendValue}s for searching in a {@link TrieSearch}.
*/
- List> getSearches(SearchConfig config, NamespaceStorage storage);
+ List> getSearches(IndexConfig config, NamespaceStorage storage);
/**
* The actual Searchables to use, if there is potential for deduplication/pooling.
+ *
* @implSpec The order of objects returned is used to also sort search results from different sources.
*/
@JsonIgnore
- default List getSearchReferences() {
+ default List> getSearchReferences() {
//Hopefully the only candidate will be Column
return List.of(this);
}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/StructureNode.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/StructureNode.java
index 9b96172fe8..42c74d823d 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/StructureNode.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/StructureNode.java
@@ -7,15 +7,15 @@
import javax.validation.Valid;
import javax.validation.constraints.NotNull;
-import com.bakdata.conquery.io.jackson.serializer.NsIdRef;
import com.bakdata.conquery.apiv1.KeyValue;
+import com.bakdata.conquery.io.jackson.serializer.NsIdRef;
import com.bakdata.conquery.models.datasets.Dataset;
import com.bakdata.conquery.models.identifiable.Labeled;
import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId;
import com.bakdata.conquery.models.identifiable.ids.specific.StructureNodeId;
import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonManagedReference;
-
+import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
@@ -32,6 +32,7 @@ public class StructureNode extends Labeled {
@Valid @JsonManagedReference(MANAGED_STRUCTURE_STRUCTURE)
private List children = Collections.emptyList();
@JsonBackReference(MANAGED_STRUCTURE_STRUCTURE)
+ @EqualsAndHashCode.Exclude
private StructureNode parent;
@Getter
private LinkedHashSet containedRoots = new LinkedHashSet<>();
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ValidityDate.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ValidityDate.java
index a827c86e47..5bdbe1c8bf 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ValidityDate.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ValidityDate.java
@@ -11,6 +11,7 @@
import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonIgnore;
import io.dropwizard.validation.ValidationMethod;
+import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@@ -26,6 +27,7 @@ public class ValidityDate extends Labeled implements NamespacedI
@NotNull
private Column column;
@JsonBackReference
+ @EqualsAndHashCode.Exclude
private Connector connector;
@Override
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java
index 98097bc40c..4447e29f9f 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java
@@ -17,6 +17,7 @@
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import io.dropwizard.validation.ValidationMethod;
+import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@@ -31,12 +32,14 @@
@JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type")
@CPSBase
@Slf4j
+@EqualsAndHashCode(callSuper = true)
public abstract class Filter extends Labeled implements NamespacedIdentifiable {
private String unit;
@JsonAlias("description")
private String tooltip;
@JsonBackReference
+ @EqualsAndHashCode.Exclude
private Connector connector;
private String pattern;
private Boolean allowDropFile;
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java
index c564d8cb2a..e8f06ecc24 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java
@@ -12,11 +12,12 @@
import com.bakdata.conquery.io.jackson.View;
import com.bakdata.conquery.io.jackson.serializer.NsIdRef;
import com.bakdata.conquery.io.storage.NamespaceStorage;
-import com.bakdata.conquery.models.config.SearchConfig;
+import com.bakdata.conquery.models.config.IndexConfig;
import com.bakdata.conquery.models.datasets.concepts.Searchable;
import com.bakdata.conquery.models.datasets.concepts.filters.SingleColumnFilter;
import com.bakdata.conquery.models.events.MajorTypeId;
import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
+import com.bakdata.conquery.models.identifiable.ids.specific.FilterId;
import com.bakdata.conquery.models.query.FilterSearch;
import com.bakdata.conquery.util.search.TrieSearch;
import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -35,7 +36,7 @@
@NoArgsConstructor
@Slf4j
@JsonIgnoreProperties({"searchType"})
-public abstract class SelectFilter extends SingleColumnFilter implements Searchable {
+public abstract class SelectFilter extends SingleColumnFilter implements Searchable {
/**
* user given mapping from the values in the CSVs to shown labels
@@ -88,8 +89,8 @@ public boolean isNotUsingTemplateAndLabels() {
private boolean generateSearchSuffixes = true;
@Override
- public List getSearchReferences() {
- final List out = new ArrayList<>();
+ public List> getSearchReferences() {
+ final List> out = new ArrayList<>();
if (getTemplate() != null) {
out.add(getTemplate());
@@ -126,9 +127,9 @@ public boolean isSearchDisabled() {
}
@Override
- public List> getSearches(SearchConfig config, NamespaceStorage storage) {
+ public List> getSearches(IndexConfig config, NamespaceStorage storage) {
- TrieSearch search = new TrieSearch<>(config.getSuffixLength(), config.getSplit());
+ TrieSearch search = new TrieSearch<>(config.getSearchSuffixLength(), config.getSearchSplitChars());
labels.entrySet()
.stream()
.map(entry -> new FrontendValue(entry.getKey(), entry.getValue()))
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java
index b66174f619..182d13ab59 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java
@@ -22,17 +22,22 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import io.dropwizard.validation.ValidationMethod;
+import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
-@JsonTypeInfo(use=JsonTypeInfo.Id.CUSTOM, property="type")
+@JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type")
@CPSBase
@Slf4j
+@EqualsAndHashCode(callSuper = true)
public abstract class Select extends Labeled implements NamespacedIdentifiable {
- @JsonBackReference @Getter @Setter
+ @EqualsAndHashCode.Exclude
+ @JsonBackReference
+ @Getter
+ @Setter
private SelectHolder holder;
@JsonIgnore
@@ -41,7 +46,8 @@ public Dataset getDataset() {
return getHolder().findConcept().getDataset();
}
- @Setter @Getter
+ @Setter
+ @Getter
private String description;
/**
diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeChild.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeChild.java
index 829eed8120..7e941a607d 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeChild.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeChild.java
@@ -14,6 +14,7 @@
import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonManagedReference;
+import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
@@ -32,6 +33,7 @@ public class ConceptTreeChild extends ConceptElement impleme
@JsonBackReference
@Getter
@Setter
+ @EqualsAndHashCode.Exclude
private ConceptTreeNode parent;
@JsonIgnore
@Getter
diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/InternalExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/InternalExecution.java
index 73c02d0ddb..f34aa3223f 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/execution/InternalExecution.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/execution/InternalExecution.java
@@ -20,4 +20,5 @@ public interface InternalExecution {
* Is called once per shard node
*/
void addResult(R result);
+
}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java
index e161f9f663..047ecc25eb 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java
@@ -16,9 +16,9 @@
import javax.validation.constraints.NotNull;
import javax.ws.rs.core.UriBuilder;
-import com.bakdata.conquery.apiv1.ExecutionStatus;
-import com.bakdata.conquery.apiv1.FullExecutionStatus;
-import com.bakdata.conquery.apiv1.OverviewExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.ExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.FullExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus;
import com.bakdata.conquery.apiv1.query.QueryDescription;
import com.bakdata.conquery.io.cps.CPSBase;
import com.bakdata.conquery.io.jackson.serializer.MetaIdRef;
@@ -52,6 +52,7 @@
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.Uninterruptibles;
import lombok.AccessLevel;
+import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NonNull;
import lombok.Setter;
@@ -66,6 +67,7 @@
@Slf4j
@CPSBase
@JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type")
+@EqualsAndHashCode(callSuper = false)
public abstract class ManagedExecution extends IdentifiableImpl implements Taggable, Shareable, Labelable, Owned, Visitable {
/**
@@ -94,28 +96,38 @@ public abstract class ManagedExecution extends IdentifiableImpl implements Sharea
@NotNull
private String[] tags = ArrayUtils.EMPTY_STRING_ARRAY;
private boolean shared = false;
-
+
/**
- * This is a blackbox for us at the moment, where the front end saves the state of the
- * formular, when the user saved it.
+ * This is a blackbox for us at the moment, where the front end saves the state of the
+ * form, when the user saved it.
*/
@NotNull
private JsonNode values;
diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQuery.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQuery.java
index b624e7dc1d..6ce35c51de 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQuery.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQuery.java
@@ -32,6 +32,16 @@
@CPSType(id = "ABSOLUTE_FORM_QUERY", base = QueryDescription.class)
public class AbsoluteFormQuery extends Query {
+ /**
+ * Index of the column, where the Resolutions name will be placed.
+ */
+ public static final int RESOLUTION_INDEX = 0;
+
+ /**
+ * Indef of the column, where the time periods will be placed.
+ */
+ public static final int TIME_INDEX = 2;
+
/**
* see {@linkplain this#getResultInfos()}.
*/
diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java
index af51219d58..4224061a7b 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java
@@ -11,8 +11,10 @@
import com.bakdata.conquery.models.query.results.MultilineEntityResult;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
+import lombok.ToString;
@Getter @RequiredArgsConstructor
+@ToString
public class AbsoluteFormQueryPlan implements QueryPlan {
private final QueryPlan query;
diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java
index 0464b9c2a3..77218e9cdf 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java
@@ -17,11 +17,13 @@
import com.bakdata.conquery.models.query.results.EntityResult;
import com.bakdata.conquery.models.query.results.MultilineEntityResult;
import lombok.RequiredArgsConstructor;
+import lombok.ToString;
/**
* Implementation of the QueryPlan for an {@link EntityDateQuery}.
*/
@RequiredArgsConstructor
+@ToString
public class EntityDateQueryPlan implements QueryPlan {
diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java
new file mode 100644
index 0000000000..ef6f840542
--- /dev/null
+++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java
@@ -0,0 +1,191 @@
+package com.bakdata.conquery.models.forms.managed;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Stream;
+
+import javax.ws.rs.core.Response;
+
+import com.bakdata.conquery.apiv1.execution.ExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.ResultAsset;
+import com.bakdata.conquery.apiv1.forms.ExternalForm;
+import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.io.external.form.ExternalFormBackendApi;
+import com.bakdata.conquery.io.external.form.ExternalTaskState;
+import com.bakdata.conquery.io.result.ExternalResult;
+import com.bakdata.conquery.io.storage.MetaStorage;
+import com.bakdata.conquery.models.auth.entities.Subject;
+import com.bakdata.conquery.models.auth.entities.User;
+import com.bakdata.conquery.models.config.FormBackendConfig;
+import com.bakdata.conquery.models.datasets.Dataset;
+import com.bakdata.conquery.models.error.ConqueryError;
+import com.bakdata.conquery.models.execution.ExecutionState;
+import com.bakdata.conquery.models.execution.ManagedExecution;
+import com.bakdata.conquery.resources.api.ResultExternalResource;
+import com.bakdata.conquery.util.AuthUtil;
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.OptBoolean;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.MoreCollectors;
+import it.unimi.dsi.fastutil.Pair;
+import lombok.EqualsAndHashCode;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * This execution type is for external form backends which use Conquery as a proxy for their task execution.
+ * An {@link ExternalForm} is wrapped in this execution to keep
+ * track of it's state using a configured API (see {@link FormBackendConfig} and {@link ExternalFormBackendApi}).
+ */
+@Slf4j
+@CPSType(id = "EXTERNAL_EXECUTION", base = ManagedExecution.class)
+@EqualsAndHashCode(callSuper = true, doNotUseGetters = true)
+public class ExternalExecution extends ManagedForm implements ExternalResult {
+
+
+ private UUID externalTaskId;
+
+ @JsonIgnore
+ @EqualsAndHashCode.Exclude
+ private ExternalFormBackendApi api;
+
+ @JsonIgnore
+ @EqualsAndHashCode.Exclude
+ private FormBackendConfig formBackendConfig;
+
+ @JsonIgnore
+ @EqualsAndHashCode.Exclude
+ private User serviceUser;
+
+
+ /**
+ * Pairs of external result assets (internal url) and their internal asset builder.
+ * The internal asset builder generates the asset url with the context of a user request.
+ */
+ @JsonIgnore
+ private List> resultsAssetMap = Collections.emptyList();
+
+ @JsonCreator
+ protected ExternalExecution(@JacksonInject(useInput = OptBoolean.FALSE) MetaStorage storage) {
+ super(storage);
+ }
+
+ public ExternalExecution(ExternalForm form, User user, Dataset dataset, MetaStorage storage) {
+ super(form, user, dataset, storage);
+ }
+
+ @Override
+ protected void doInitExecutable() {
+ formBackendConfig = getConfig().getPluginConfigs(FormBackendConfig.class)
+ .filter(c -> c.supportsFormType(getSubmittedForm().getFormType()))
+ .collect(MoreCollectors.onlyElement());
+
+ api = formBackendConfig.createApi();
+ }
+
+ @Override
+ public void start() {
+
+ synchronized (this) {
+
+ if (externalTaskId != null) {
+ syncExternalState();
+ }
+
+ if (getState() == ExecutionState.RUNNING) {
+ throw new ConqueryError.ExecutionProcessingError();
+ }
+
+ super.start();
+
+ // Create service user
+ serviceUser = formBackendConfig.createServiceUser(getOwner(), getDataset());
+
+ final ExternalTaskState externalTaskState = api.postForm(getSubmitted(), getOwner(), serviceUser, getDataset());
+
+ externalTaskId = externalTaskState.getId();
+ }
+ }
+
+ private synchronized void syncExternalState() {
+ Preconditions.checkNotNull(externalTaskId, "Cannot check external task, because no Id is present");
+
+ final ExternalTaskState formState = api.getFormState(externalTaskId);
+
+ switch (formState.getStatus()) {
+
+ case RUNNING -> {
+ setState(ExecutionState.RUNNING);
+ setProgress(formState.getProgress().floatValue());
+ }
+ case FAILURE -> fail(formState.getError());
+ case SUCCESS -> {
+ resultsAssetMap = registerResultAssets(formState);
+ finish(ExecutionState.DONE);
+ }
+ }
+ }
+
+ private List> registerResultAssets(ExternalTaskState response) {
+ final List> assetMap = new ArrayList<>();
+ response.getResults().forEach(asset ->
+ {
+ assetMap.add(Pair.of(asset, createResultAssetBuilder(asset)));
+ });
+ return assetMap;
+ }
+
+ /**
+ * The {@link ResultAsset} is request-dependent, so we can prepare only builder here which takes an url builder.
+ */
+ private AssetBuilder createResultAssetBuilder(ResultAsset asset) {
+ return (uriBuilder) -> {
+ try {
+ final URI externalDownloadURL = ResultExternalResource.getDownloadURL(uriBuilder.clone(), this, asset.getAssetId());
+ return new ResultAsset(asset.label(), externalDownloadURL);
+ }
+ catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ };
+ }
+
+ @Override
+ public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus status) {
+ syncExternalState();
+
+ super.setStatusBase(subject, status);
+ }
+
+ @Override
+ public Stream getResultAssets() {
+ return resultsAssetMap.stream().map(Pair::value);
+ }
+
+ @Override
+ public Response fetchExternalResult(String assetId) {
+ final ResultAsset resultRef = resultsAssetMap.stream()
+ .map(Pair::key).filter(a -> a.getAssetId().equals(assetId))
+ .collect(MoreCollectors.onlyElement());
+
+ return api.getResult(resultRef.url());
+ }
+
+ @Override
+ protected void finish(ExecutionState executionState) {
+ if (getState().equals(executionState)) {
+ return;
+ }
+ super.finish(executionState);
+ synchronized (this) {
+ AuthUtil.cleanUpUserAndBelongings(serviceUser, getStorage());
+ serviceUser = null;
+ }
+ }
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java
index 321b70e805..3b5df25c50 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java
@@ -5,7 +5,6 @@
import java.util.Optional;
import java.util.OptionalInt;
-import com.bakdata.conquery.apiv1.forms.FeatureGroup;
import com.bakdata.conquery.models.common.CDateSet;
import com.bakdata.conquery.models.forms.util.DateContext;
import com.bakdata.conquery.models.forms.util.Resolution;
@@ -19,8 +18,12 @@
import com.bakdata.conquery.models.query.results.SinglelineEntityResult;
import com.bakdata.conquery.util.QueryUtils;
import lombok.Getter;
+import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
@Getter
+@ToString
+@Slf4j
public class FormQueryPlan implements QueryPlan {
private final List dateContexts;
@@ -37,7 +40,8 @@ public FormQueryPlan(List dateContexts, ArrayConceptQueryPlan featu
if (dateContexts.size() <= 0) {
- // There is nothing to do for this FormQueryPlan but we will return an empty result when its executed
+ // There is nothing to do for this FormQueryPlan, but we will return an empty result when its executed
+ log.warn("dateContexts are empty. Will not produce a result.");
constantCount = 3;
withRelativeEventDate = false;
return;
diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java
index 63b8001e0b..410460a5ac 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java
@@ -4,7 +4,6 @@
import com.bakdata.conquery.apiv1.forms.Form;
import com.bakdata.conquery.apiv1.forms.FormConfigAPI;
-import com.bakdata.conquery.apiv1.query.QueryDescription;
import com.bakdata.conquery.io.cps.CPSType;
import com.bakdata.conquery.io.storage.MetaStorage;
import com.bakdata.conquery.models.auth.entities.User;
@@ -16,41 +15,44 @@
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.OptBoolean;
+import com.fasterxml.jackson.databind.DatabindContext;
import lombok.EqualsAndHashCode;
import lombok.Getter;
-import lombok.Setter;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
/**
* Internal runtime representation of a form query.
*/
-@Getter
-@Setter
@ToString
@Slf4j
@EqualsAndHashCode(callSuper = true)
@CPSType(id = "MANAGED_FORM", base = ManagedExecution.class)
-public class ManagedForm extends ManagedExecution {
+public abstract class ManagedForm extends ManagedExecution {
/**
- * The form that was submitted through the api.
+ * The submitted form for this execution.
+ *
+ * @implNote We use the type {@link Form} here rather than the type parameter F.
+ * Using F causes the class to have a concrete type at runtime which in turn skips
+ * the object inspection of Jackson to look at the actual type member of the object (see {@link com.bakdata.conquery.io.cps.CPSTypeIdResolver#typeFromId(DatabindContext, String)}).
+ * This causes a problem, when the object uses types with {@link com.bakdata.conquery.io.cps.SubTyped},
+ * as the subtype is only added to the {@link com.fasterxml.jackson.databind.DeserializationContext}, when the
+ * type is derived from the type
member not when Jackson can just infer the deserializer from the type of
+ * this property.
*/
- private F submittedForm;
+ @Getter
+ private Form submittedForm;
protected ManagedForm(@JacksonInject(useInput = OptBoolean.FALSE) MetaStorage storage) {
super(storage);
}
- public ManagedForm(F submittedForm, User owner, Dataset submittedDataset, MetaStorage storage) {
+ protected ManagedForm(F submittedForm, User owner, Dataset submittedDataset, MetaStorage storage) {
super(owner, submittedDataset, storage);
this.submittedForm = submittedForm;
}
- @Override
- protected void doInitExecutable() {
-
- }
@Override
public void start() {
@@ -79,8 +81,8 @@ public void visit(Consumer visitor) {
@Override
@JsonIgnore
- public QueryDescription getSubmitted() {
- return submittedForm;
+ public F getSubmitted() {
+ return (F) submittedForm;
}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java
index 807802d509..67c15c0864 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java
@@ -5,7 +5,7 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import com.bakdata.conquery.apiv1.FullExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.FullExecutionStatus;
import com.bakdata.conquery.apiv1.forms.Form;
import com.bakdata.conquery.apiv1.forms.InternalForm;
import com.bakdata.conquery.io.cps.CPSType;
@@ -41,6 +41,7 @@
@Slf4j
@CPSType(base = ManagedExecution.class, id = "INTERNAL_FORM")
@Getter
+@EqualsAndHashCode(callSuper = true)
public class ManagedInternalForm extends ManagedForm implements SingleTableResult, InternalExecution {
@@ -49,10 +50,11 @@ public class ManagedInternalForm extends ManagedF
* This is required by forms that have multiple results (CSVs) as output.
*/
@JsonIgnore
+ @EqualsAndHashCode.Exclude
private Map subQueries;
/**
- * Subqueries that are send to the workers.
+ * Subqueries that are sent to the workers.
*/
@JsonIgnore
@EqualsAndHashCode.Exclude
@@ -69,7 +71,7 @@ public ManagedInternalForm(F form, User user, Dataset submittedDataset, MetaStor
@Override
public void doInitExecutable() {
// Convert sub queries to sub executions
- getSubmittedForm().resolve(new QueryResolveContext(getNamespace(), getConfig(), getStorage(), null));
+ getSubmitted().resolve(new QueryResolveContext(getNamespace(), getConfig(), getStorage(), null));
subQueries = createSubExecutions();
// Initialize sub executions
@@ -78,9 +80,9 @@ public void doInitExecutable() {
@NotNull
private Map createSubExecutions() {
- return getSubmittedForm().createSubQueries()
- .entrySet()
- .stream().collect(Collectors.toMap(
+ return getSubmitted().createSubQueries()
+ .entrySet()
+ .stream().collect(Collectors.toMap(
e -> e.getKey(),
e -> e.getValue().toManagedExecution(getOwner(), getDataset(), getStorage())
@@ -103,9 +105,7 @@ public List generateColumnDescriptions() {
}
- @Override
protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) {
- super.setAdditionalFieldsForStatusWithColumnDescription(subject, status);
// Set the ColumnDescription if the Form only consits of a single subquery
if (subQueries == null) {
// If subqueries was not set the Execution was not initialized, do it manually
diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java
index ea819fb08c..cb074823f2 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java
@@ -26,11 +26,13 @@
import com.google.common.collect.ImmutableList;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
+import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Getter
@RequiredArgsConstructor
+@ToString
public class RelativeFormQueryPlan implements QueryPlan {
// Position of fixed columns in the result. (This is without identifier column[s], they are added upon result rendering)
@@ -80,7 +82,7 @@ public Optional execute(QueryExecutionContext ctx, Entity
int size = calculateCompleteLength();
EntityResult contained = preResult.get();
// Gather all validity dates from prerequisite
- CDateSet dateSet = query.getValidityDateAggregator().map(Aggregator::createAggregationResult).orElseGet(CDateSet::create);
+ CDateSet dateSet = query.getValidityDateAggregator().map(Aggregator::createAggregationResult).orElseGet(CDateSet::createEmpty);
final OptionalInt sampled = indexSelector.sample(dateSet);
diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ManagedExecutionId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ManagedExecutionId.java
index 32671646eb..f0a2d62e40 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ManagedExecutionId.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/specific/ManagedExecutionId.java
@@ -13,7 +13,7 @@
@AllArgsConstructor
@Getter
-@EqualsAndHashCode(callSuper = false)
+@EqualsAndHashCode(callSuper = false, doNotUseGetters = true)
public class ManagedExecutionId extends Id {
private final DatasetId dataset;
diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java
index 3eb95e950f..dff8a9996f 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java
@@ -1,5 +1,6 @@
package com.bakdata.conquery.models.index;
+import java.net.URI;
import java.net.URL;
import lombok.Data;
@@ -11,6 +12,6 @@
*/
@Data
public abstract class AbstractIndexKey>> implements IndexKey {
- private final URL csv;
+ private final URI csv;
private final String internalColumn;
}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java
index a86928219b..5ca864c5ba 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java
@@ -1,5 +1,6 @@
package com.bakdata.conquery.models.index;
+import java.net.URI;
import java.net.URL;
import java.util.List;
@@ -28,7 +29,7 @@ public class FrontendValueIndexKey extends AbstractIndexKey
*/
private final String optionValueTemplate;
- public FrontendValueIndexKey(URL csv, String internalColumn, String valueTemplate, String optionValueTemplate, int suffixCutoff, String splitPattern) {
+ public FrontendValueIndexKey(URI csv, String internalColumn, String valueTemplate, String optionValueTemplate, int suffixCutoff, String splitPattern) {
super(csv, internalColumn);
this.suffixCutoff = suffixCutoff;
this.splitPattern = splitPattern;
diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java
index 3875c58643..fda0bedaf1 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java
@@ -1,5 +1,6 @@
package com.bakdata.conquery.models.index;
+import java.net.URI;
import java.net.URL;
import java.util.List;
@@ -11,7 +12,14 @@
* @param The type of Index that is indexed by this key
*/
public interface IndexKey>> {
- URL getCsv();
+
+ /**
+ * An url, or a part of it, that points to the referenced csv file.
+ *
+ * @implNote This is an url but implemented as an uri in this data object, because url can have undesired
+ * side effects: URL equals() and hashcode()
+ */
+ URI getCsv();
String getInternalColumn();
diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java
index a22e7b7c50..a8c3e9bd54 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java
@@ -45,7 +45,7 @@ public Index load(@NotNull IndexKey key) throws Exception {
final CsvParser csvParser = new CsvParser(csvParserSettings);
- try (InputStream inputStream = key.getCsv().openStream()) {
+ try (InputStream inputStream = key.getCsv().toURL().openStream()) {
final IterableResult records = csvParser.iterateRecords(inputStream);
diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java
index 836016711a..cdaaad9619 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java
@@ -1,5 +1,6 @@
package com.bakdata.conquery.models.index;
+import java.net.URI;
import java.net.URL;
import java.util.List;
@@ -12,7 +13,7 @@ public class MapIndexKey extends AbstractIndexKey {
private final String externalTemplate;
- public MapIndexKey(URL csv, String internalColumn, String externalTemplate) {
+ public MapIndexKey(URI csv, String internalColumn, String externalTemplate) {
super(csv, internalColumn);
this.externalTemplate = externalTemplate;
}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java
index 238c1708a8..0ee8181697 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java
@@ -1,17 +1,19 @@
package com.bakdata.conquery.models.index;
-import java.net.URL;
+import java.net.URI;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import com.bakdata.conquery.io.cps.CPSType;
import com.bakdata.conquery.io.jackson.serializer.NsIdRef;
+import com.bakdata.conquery.models.config.ConqueryConfig;
import com.bakdata.conquery.models.datasets.Dataset;
import com.bakdata.conquery.models.identifiable.NamedImpl;
import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable;
import com.bakdata.conquery.models.identifiable.ids.specific.InternToExternMapperId;
+import com.bakdata.conquery.util.io.FileUtil;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.OptBoolean;
@@ -37,6 +39,10 @@ public class MapInternToExternMapper extends NamedImpl i
@JacksonInject(useInput = OptBoolean.FALSE)
private IndexService mapIndex;
+ @JsonIgnore
+ @JacksonInject(useInput = OptBoolean.FALSE)
+ private ConqueryConfig config;
+
@NsIdRef
@Setter
@NotNull
@@ -47,7 +53,7 @@ public class MapInternToExternMapper extends NamedImpl i
private final String name;
@ToString.Include
@NotNull
- private final URL csv;
+ private final URI csv;
@ToString.Include
@NotEmpty
private final String internalColumn;
@@ -64,9 +70,14 @@ public class MapInternToExternMapper extends NamedImpl i
@Override
public synchronized void init() {
- int2ext = mapIndex.getIndex(new MapIndexKey(csv, internalColumn, externalTemplate));
+
+ final URI resolvedURI = FileUtil.getResolvedUri(config.getIndex().getBaseUrl(), csv);
+ log.trace("Resolved mapping reference csv url '{}': {}", this.getId(), resolvedURI);
+
+ int2ext = mapIndex.getIndex(new MapIndexKey(resolvedURI, internalColumn, externalTemplate));
}
+
@Override
public boolean initialized() {
return int2ext != null;
diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java
index 12fc0ccb06..2b315e92bc 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java
@@ -12,10 +12,11 @@
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import com.bakdata.conquery.apiv1.frontend.FrontendValue;
import com.bakdata.conquery.io.storage.NamespaceStorage;
-import com.bakdata.conquery.models.config.SearchConfig;
+import com.bakdata.conquery.models.config.IndexConfig;
import com.bakdata.conquery.models.datasets.concepts.Searchable;
import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter;
import com.bakdata.conquery.util.search.TrieSearch;
@@ -34,13 +35,13 @@ public class UpdateFilterSearchJob extends Job {
private final NamespaceStorage storage;
@NonNull
- private final Map> searchCache;
+ private final Map, TrieSearch> searchCache;
@NonNull
- private final SearchConfig searchConfig;
+ private final IndexConfig indexConfig;
@NonNull
- private final Object2LongMap> totals;
+ private final Object2LongMap> totals;
@Override
public void execute() throws Exception {
@@ -58,7 +59,7 @@ public void execute() throws Exception {
.collect(Collectors.toList());
- final Set collectedSearchables =
+ final Set> collectedSearchables =
allSelectFilters.stream()
.map(SelectFilter::getSearchReferences)
.flatMap(Collection::stream)
@@ -71,12 +72,11 @@ public void execute() throws Exception {
// Most computations are cheap but data intensive: we fork here to use as many cores as possible.
final ExecutorService service = Executors.newCachedThreadPool();
- final Map> synchronizedResult = Collections.synchronizedMap(searchCache);
+ final Map, TrieSearch> synchronizedResult = Collections.synchronizedMap(searchCache);
log.debug("Found {} searchable Objects.", collectedSearchables.size());
-
- for (Searchable searchable : collectedSearchables) {
+ for (Searchable searchable : collectedSearchables) {
service.submit(() -> {
@@ -85,7 +85,7 @@ public void execute() throws Exception {
log.info("BEGIN collecting entries for `{}`", searchable);
try {
- final List> values = searchable.getSearches(searchConfig, storage);
+ final List> values = searchable.getSearches(indexConfig, storage);
for (TrieSearch search : values) {
synchronizedResult.put(searchable, search);
@@ -121,17 +121,23 @@ public void execute() throws Exception {
// Precompute totals as that can be slow when doing it on-demand.
totals.putAll(
- allSelectFilters.parallelStream()
- .collect(Collectors.toMap(
- Functions.identity(),
- filter -> filter.getSearchReferences().stream()
- .map(searchCache::get)
- .filter(Objects::nonNull) // Failed or disabled searches are null
- .flatMap(TrieSearch::stream)
- .mapToInt(FrontendValue::hashCode)
- .distinct()
- .count()
- ))
+ Stream.concat(
+ // SelectFilters without their own labels are not "real" Searchables and therefore not in collectedSearchables
+ // We however want the real totals of ALL Searchables (and especially SelectFilters), which is why we include them here explicitly
+ allSelectFilters.parallelStream(),
+ collectedSearchables.parallelStream()
+ )
+ .distinct()
+ .collect(Collectors.toMap(
+ Functions.identity(),
+ filter -> filter.getSearchReferences().stream()
+ .map(searchCache::get)
+ .filter(Objects::nonNull) // Failed or disabled searches are null
+ .flatMap(TrieSearch::stream)
+ .mapToInt(FrontendValue::hashCode)
+ .distinct()
+ .count()
+ ))
);
diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/package-info.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/package-info.java
index f911a37573..b88132ae4f 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/preproc/package-info.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/package-info.java
@@ -1,18 +1,18 @@
package com.bakdata.conquery.models.preproc;
/**
* Note on the CQPP file format:
- *
+ *
* It is encoded as Smile/BinaryJson-format consisting of three documents:
* - {@link com.bakdata.conquery.models.preproc.PreprocessedHeader}: metadata of the import.
* - {@link com.bakdata.conquery.models.preproc.PreprocessedDictionaries}: dictionary encoded strings for the import.
* - {@link com.bakdata.conquery.models.preproc.PreprocessedData}: the description and raw representation of the data as {@link com.bakdata.conquery.models.events.stores.root.ColumnStore}.
- *
+ *
* The file is split into three sections, so we can load them progressively:
* Initially, we just read the header and determine if it isn't already loaded, and also fits to the {@link com.bakdata.conquery.models.datasets.Table} it is supposed to go in.
* We then submit an {@link com.bakdata.conquery.models.jobs.ImportJob} which will load the data.
* First the {@link com.bakdata.conquery.models.dictionary.Dictionary}s. Those are imported and are potentially altered or ingested into shared-Dictionaries (via {@link com.bakdata.conquery.models.datasets.Column#getSharedDictionary()}).
- *
+ *
* We then load the raw data, having claims for Dictionaries in the import resolved via {@link com.bakdata.conquery.io.jackson.serializer.NsIdRef}, which is why they need to be loaded in a second step.
- *
+ *
* TODO write the rest of the documentation for {@link com.bakdata.conquery.models.jobs.ImportJob}
*/
\ No newline at end of file
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ColumnDescriptor.java b/backend/src/main/java/com/bakdata/conquery/models/query/ColumnDescriptor.java
index 0074db2924..418153771b 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/ColumnDescriptor.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/ColumnDescriptor.java
@@ -7,18 +7,14 @@
import com.bakdata.conquery.models.types.SemanticType;
import lombok.AllArgsConstructor;
import lombok.Builder;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.ToString;
+import lombok.Data;
/**
* Container class for the query API provide meta data for reach column in the
* csv result. This can be used for the result preview in the frontend.
*/
-@Getter
+@Data
@Builder
-@ToString
-@NoArgsConstructor
@AllArgsConstructor
public class ColumnDescriptor {
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java
index 1f4f9de287..63850ff7e2 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java
@@ -10,7 +10,7 @@
import com.bakdata.conquery.apiv1.frontend.FrontendValue;
import com.bakdata.conquery.io.storage.NamespaceStorage;
import com.bakdata.conquery.models.config.CSVConfig;
-import com.bakdata.conquery.models.config.SearchConfig;
+import com.bakdata.conquery.models.config.IndexConfig;
import com.bakdata.conquery.models.datasets.concepts.Searchable;
import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter;
import com.bakdata.conquery.models.jobs.JobManager;
@@ -32,7 +32,7 @@ public class FilterSearch {
private final NamespaceStorage storage;
private final JobManager jobManager;
private final CSVConfig parserConfig;
- private final SearchConfig searchConfig;
+ private final IndexConfig indexConfig;
/**
* We tag our searches based on references collected in getSearchReferences. We do not mash them all together to allow for sharing and prioritising different sources.
@@ -40,14 +40,14 @@ public class FilterSearch {
* In the code below, the keys of this map will usually be called "reference".
*/
@JsonIgnore
- private final Map> searchCache = new HashMap<>();
- private Object2LongMap> totals = Object2LongMaps.emptyMap();
+ private final Map, TrieSearch> searchCache = new HashMap<>();
+ private Object2LongMap> totals = Object2LongMaps.emptyMap();
/**
* From a given {@link FrontendValue} extract all relevant keywords.
*/
public static List extractKeywords(FrontendValue value) {
- List keywords = new ArrayList<>(3);
+ final List keywords = new ArrayList<>(3);
keywords.add(value.getLabel());
keywords.add(value.getValue());
@@ -62,15 +62,15 @@ public static List extractKeywords(FrontendValue value) {
/**
* For a {@link SelectFilter} collect all relevant {@link TrieSearch}.
*/
- public List> getSearchesFor(SelectFilter filter) {
- return filter.getSearchReferences().stream()
- .map(searchCache::get)
- .filter(Objects::nonNull)
- .collect(Collectors.toList());
+ public List> getSearchesFor(Searchable searchable) {
+ return searchable.getSearchReferences().stream()
+ .map(searchCache::get)
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList());
}
- public long getTotal(SelectFilter filter) {
- return totals.getOrDefault(filter, 0);
+ public long getTotal(Searchable searchable) {
+ return totals.getOrDefault(searchable, 0);
}
@@ -81,7 +81,7 @@ public void updateSearch() {
totals = new Object2LongOpenHashMap<>();
- jobManager.addSlowJob(new UpdateFilterSearchJob(storage, searchCache, searchConfig, totals));
+ jobManager.addSlowJob(new UpdateFilterSearchJob(storage, searchCache, indexConfig, totals));
}
}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java
index 7bb935af18..4bd1f5449a 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java
@@ -13,8 +13,8 @@
import java.util.stream.Stream;
import c10n.C10N;
-import com.bakdata.conquery.apiv1.ExecutionStatus;
-import com.bakdata.conquery.apiv1.FullExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.ExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.FullExecutionStatus;
import com.bakdata.conquery.apiv1.query.Query;
import com.bakdata.conquery.apiv1.query.QueryDescription;
import com.bakdata.conquery.apiv1.query.SecondaryIdQuery;
@@ -146,9 +146,7 @@ public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus sta
}
}
- @Override
protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) {
- super.setAdditionalFieldsForStatusWithColumnDescription(subject, status);
if (columnDescriptions == null) {
columnDescriptions = generateColumnDescriptions();
}
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java
index 4c771e2de7..2d61ca4d9c 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/QueryPlanContext.java
@@ -23,7 +23,6 @@ public class QueryPlanContext {
private CDateRange dateRestriction = CDateRange.all();
-
/**
* Set if in {@link com.bakdata.conquery.models.query.queryplan.SecondaryIdQueryPlan}, to the query-active {@link SecondaryIdDescriptionId}.
*/
diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java
index 170515555b..5e3de93ae2 100644
--- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java
+++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java
@@ -1,23 +1,31 @@
package com.bakdata.conquery.models.query.preview;
+import java.time.LocalDate;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import com.bakdata.conquery.apiv1.FullExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.FullExecutionStatus;
import com.bakdata.conquery.io.cps.CPSType;
import com.bakdata.conquery.io.storage.MetaStorage;
import com.bakdata.conquery.models.auth.entities.Subject;
import com.bakdata.conquery.models.auth.entities.User;
-import com.bakdata.conquery.models.config.ConqueryConfig;
+import com.bakdata.conquery.models.common.CDate;
+import com.bakdata.conquery.models.common.QuarterUtils;
import com.bakdata.conquery.models.datasets.Dataset;
import com.bakdata.conquery.models.datasets.PreviewConfig;
import com.bakdata.conquery.models.execution.ManagedExecution;
import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery;
import com.bakdata.conquery.models.forms.managed.ManagedInternalForm;
+import com.bakdata.conquery.models.forms.util.Resolution;
import com.bakdata.conquery.models.i18n.I18n;
+import com.bakdata.conquery.models.identifiable.ids.specific.SelectId;
import com.bakdata.conquery.models.messages.namespaces.WorkerMessage;
import com.bakdata.conquery.models.messages.namespaces.specific.ExecuteForm;
import com.bakdata.conquery.models.query.ColumnDescriptor;
@@ -25,43 +33,82 @@
import com.bakdata.conquery.models.query.PrintSettings;
import com.bakdata.conquery.models.query.SingleTableResult;
import com.bakdata.conquery.models.query.resultinfo.ResultInfo;
+import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo;
import com.bakdata.conquery.models.query.results.EntityResult;
import com.bakdata.conquery.models.query.results.MultilineEntityResult;
import com.bakdata.conquery.models.types.SemanticType;
-import com.bakdata.conquery.models.worker.Namespace;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.OptBoolean;
import com.google.common.collect.MoreCollectors;
+import lombok.ToString;
+import org.apache.logging.log4j.util.Strings;
+import org.jetbrains.annotations.NotNull;
/**
* Dedicated {@link ManagedExecution} to properly display/combine the two Queries submitted by {@link EntityPreviewForm}.
* This mostly delegates to {@link EntityPreviewForm#VALUES_QUERY_NAME}, but embeds the result of {@link EntityPreviewForm#INFOS_QUERY_NAME} into {@link EntityPreviewStatus#getInfos()}.
*/
@CPSType(id = "ENTITY_PREVIEW_EXECUTION", base = ManagedExecution.class)
-public class EntityPreviewExecution extends ManagedInternalForm implements SingleTableResult {
+@ToString
+public class EntityPreviewExecution extends ManagedInternalForm {
+ @ToString.Exclude
private PreviewConfig previewConfig;
+ protected EntityPreviewExecution(@JacksonInject(useInput = OptBoolean.FALSE) MetaStorage storage) {
+ super(storage);
+ }
+
+ public EntityPreviewExecution(EntityPreviewForm entityPreviewQuery, User user, Dataset submittedDataset, MetaStorage storage) {
+ super(entityPreviewQuery, user, submittedDataset, storage);
+ }
+
@Override
public boolean isSystem() {
// This Form should NEVER be started manually. Nor persisted
return true;
}
- protected EntityPreviewExecution(@JacksonInject(useInput = OptBoolean.FALSE) MetaStorage storage) {
- super(storage);
+ @Override
+ public void doInitExecutable() {
+ super.doInitExecutable();
+ previewConfig = getNamespace().getPreviewConfig();
}
- public EntityPreviewExecution(EntityPreviewForm entityPreviewQuery, User user, Dataset submittedDataset, MetaStorage storage) {
- super(entityPreviewQuery, user, submittedDataset, storage);
+ /**
+ * Collects status of {@link EntityPreviewForm#getValuesQuery()} and {@link EntityPreviewForm#getInfoCardQuery()}.
+ *
+ * Most importantly to {@link EntityPreviewStatus#setInfos(List)} to for infos of entity.
+ */
+ @Override
+ public FullExecutionStatus buildStatusFull(Subject subject) {
+
+ initExecutable(getNamespace(), getConfig());
+
+ final EntityPreviewStatus status = new EntityPreviewStatus();
+ setStatusFull(status, subject);
+ status.setQuery(getValuesQuery().getQuery());
+
+ final PrintSettings printSettings = new PrintSettings(true, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel);
+
+ status.setInfos(transformQueryResultToInfos(getInfoCardExecution(), printSettings));
+
+ status.setTimeStratifiedInfos(toChronoInfos(previewConfig, getSubQueries(), printSettings));
+
+ return status;
+ }
+
+ @JsonIgnore
+ private ManagedQuery getValuesQuery() {
+ return getSubQueries().get(EntityPreviewForm.VALUES_QUERY_NAME);
}
/**
* Takes a ManagedQuery, and transforms its result into a List of {@link EntityPreviewStatus.Info}.
* The format of the query is an {@link AbsoluteFormQuery} containing a single line for one person. This should correspond to {@link EntityPreviewForm#VALUES_QUERY_NAME}.
*/
- private List transformQueryResultToInfos(ManagedQuery infoCardExecution, Namespace namespace, ConqueryConfig config) {
+ private List transformQueryResultToInfos(ManagedQuery infoCardExecution, PrintSettings printSettings) {
// Submitted Query is a single line of an AbsoluteFormQuery => MultilineEntityResult with a single line.
@@ -69,7 +116,6 @@ private List transformQueryResultToInfos(ManagedQuery
final Object[] values = result.getValues().get(0);
final List extraInfos = new ArrayList<>(values.length);
- final PrintSettings printSettings = new PrintSettings(true, I18n.LOCALE.get(), namespace, config, null, previewConfig::resolveSelectLabel);
// We are only interested in the Select results.
for (int index = AbsoluteFormQuery.FEATURES_OFFSET; index < infoCardExecution.getResultInfos().size(); index++) {
@@ -89,51 +135,155 @@ private List transformQueryResultToInfos(ManagedQuery
return extraInfos;
}
- @Override
- public void doInitExecutable() {
- super.doInitExecutable();
- previewConfig = getNamespace().getPreviewConfig();
+ @JsonIgnore
+ private ManagedQuery getInfoCardExecution() {
+ return getSubQueries().get(EntityPreviewForm.INFOS_QUERY_NAME);
}
- /**
- * Collects status of {@link EntityPreviewForm#getValuesQuery()} and {@link EntityPreviewForm#getInfoCardQuery()}.
- *
- * Most importantly to {@link EntityPreviewStatus#setInfos(List)} to for infos of entity.
- */
- @Override
- public FullExecutionStatus buildStatusFull(Subject subject) {
+ @NotNull
+ private List toChronoInfos(PreviewConfig previewConfig, Map subQueries, PrintSettings printSettings) {
+ final List timeStratifiedInfos = new ArrayList<>();
- initExecutable(getNamespace(), getConfig());
+ for (PreviewConfig.TimeStratifiedSelects description : previewConfig.getTimeStratifiedSelects()) {
+ final ManagedQuery query = subQueries.get(description.label());
- final EntityPreviewStatus status = new EntityPreviewStatus();
- setStatusFull(status, subject);
- status.setQuery(getValuesQuery().getQuery());
+ final EntityResult entityResult = query.streamResults().collect(MoreCollectors.onlyElement());
- status.setInfos(transformQueryResultToInfos(getInfoCardExecution(), getNamespace(), getConfig()));
+ final Map select2desc =
+ description.selects().stream()
+ .collect(Collectors.toMap(PreviewConfig.InfoCardSelect::select, Function.identity()));
- return status;
+ // Group lines by year and quarter.
+ final List yearEntries = createYearEntries(entityResult, query.getResultInfos(), printSettings, select2desc);
+
+ // get descriptions, but drop everything that isn't a select result as the rest is already structured
+ final List columnDescriptors = createChronoColumnDescriptors(query, select2desc);
+
+ final EntityPreviewStatus.TimeStratifiedInfos infos =
+ new EntityPreviewStatus.TimeStratifiedInfos(description.label(), description.description(), columnDescriptors, yearEntries);
+
+ timeStratifiedInfos.add(infos);
+ }
+
+ return timeStratifiedInfos;
}
- @Override
- protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) {
- status.setColumnDescriptions(generateColumnDescriptions());
+ @NotNull
+ private List createYearEntries(EntityResult entityResult, List resultInfos, PrintSettings printSettings, Map select2desc) {
+ final Map yearLines = new HashMap<>();
+ final Map> quarterLines = new HashMap<>();
+
+ groupLinesForResolutions(entityResult, yearLines, quarterLines);
+
+ final Function> lineTransformer = createLineTransformer(resultInfos, select2desc, printSettings);
+
+ final List yearEntries = new ArrayList<>();
+
+ yearLines.forEach((year, yearLine) -> {
+
+ final List quarterEntries = new ArrayList<>();
+ quarterLines.getOrDefault(year, Collections.emptyMap())
+ .forEach((quarter, line) -> quarterEntries.add(new EntityPreviewStatus.QuarterEntry(quarter, lineTransformer.apply(line))));
+
+ quarterEntries.sort(Comparator.comparingInt(EntityPreviewStatus.QuarterEntry::quarter));
+
+ yearEntries.add(new EntityPreviewStatus.YearEntry(year, lineTransformer.apply(yearLine), quarterEntries));
+ });
+
+ yearEntries.sort(Comparator.comparingInt(EntityPreviewStatus.YearEntry::year));
+
+ return yearEntries;
}
- @JsonIgnore
- private ManagedQuery getInfoCardExecution() {
- return getSubQueries().get(EntityPreviewForm.INFOS_QUERY_NAME);
+ /**
+ * Query contains both YEARS and QUARTERS lines: Group them.
+ */
+ private static void groupLinesForResolutions(EntityResult entityResult, Map yearLines, Map> quarterLines) {
+ for (Object[] line : entityResult.listResultLines()) {
+
+ // Since we know the dates are always aligned we need to only respect their starts.
+ final LocalDate date = CDate.toLocalDate(((List) line[AbsoluteFormQuery.TIME_INDEX]).get(0));
+
+ final int year = date.getYear();
+ final int quarter = QuarterUtils.getQuarter(date);
+
+ switch (Resolution.valueOf((String) line[AbsoluteFormQuery.RESOLUTION_INDEX])) {
+ case YEARS -> yearLines.put(year, line);
+ case QUARTERS -> quarterLines.computeIfAbsent(year, (ignored) -> new HashMap<>(4)).put(quarter, line);
+ default -> throw new IllegalStateException("Query may only have modes for Quarter and/or Year.");
+ }
+ }
}
- @JsonIgnore
- private ManagedQuery getValuesQuery() {
- return getSubQueries().get(EntityPreviewForm.VALUES_QUERY_NAME);
+ /**
+ * Creates a transformer printing lines, transformed into a Map of label->value.
+ * Null values are omitted.
+ */
+ private static Function> createLineTransformer(List resultInfos, Map select2desc, PrintSettings printSettings) {
+
+
+ final int size = resultInfos.size();
+ final String[] columnNames = new String[size];
+
+ for (int index = 0; index < size; index++) {
+ final ResultInfo resultInfo = resultInfos.get(index);
+
+ if (resultInfo instanceof SelectResultInfo selectResultInfo) {
+ columnNames[index] = select2desc.get(selectResultInfo.getSelect().getId()).label();
+ }
+ }
+
+ return line -> {
+ final Map out = new HashMap<>(size);
+
+ for (int column = 0; column < size; column++) {
+ final String columnName = columnNames[column];
+
+ if (columnName == null) {
+ continue;
+ }
+
+ final String value = resultInfos.get(column).getType().printNullable(printSettings, line[column]);
+
+ if (Strings.isBlank(value)) {
+ continue;
+ }
+
+ out.put(columnName, value);
+ }
+
+ return out;
+ };
}
+ /**
+ * For the selects in result infos, build ColumnDescriptors using definitions (label and description) from PreviewConfig.
+ */
+ private static List createChronoColumnDescriptors(SingleTableResult query, Map select2desc) {
+
+ final List columnDescriptions = new ArrayList<>();
+
+ for (ResultInfo info : query.getResultInfos()) {
+ if (info instanceof SelectResultInfo selectResultInfo) {
+ final PreviewConfig.InfoCardSelect desc = select2desc.get(selectResultInfo.getSelect().getId());
+
+ // We build these by hand because they are labeled and described by config.
+ columnDescriptions.add(ColumnDescriptor.builder()
+ .label(desc.label())
+ .defaultLabel(desc.label())
+ .type(info.getType().typeInfo())
+ .semantics(info.getSemantics())
+ .description(desc.description())
+ .build());
+ }
+ }
- @Override
- public WorkerMessage createExecutionMessage() {
- return new ExecuteForm(getId(), getFlatSubQueries().entrySet().stream()
- .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getQuery())));
+
+ return columnDescriptions;
+ }
+
+ protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) {
+ status.setColumnDescriptions(generateColumnDescriptions());
}
@Override
@@ -141,6 +291,7 @@ public List generateColumnDescriptions() {
final List descriptors = getValuesQuery().generateColumnDescriptions();
for (ColumnDescriptor descriptor : descriptors) {
+ // Add grouping semantics to secondaryIds to group by
if (descriptor.getSemantics()
.stream()
.anyMatch(semanticType -> semanticType instanceof SemanticType.SecondaryIdT desc
@@ -148,6 +299,7 @@ public List generateColumnDescriptions() {
descriptor.getSemantics().add(new SemanticType.GroupT());
}
+ // Add hidden semantics to fields flagged for hiding.
if (descriptor.getSemantics()
.stream()
.anyMatch(semanticType -> semanticType instanceof SemanticType.ColumnT desc && previewConfig.isHidden(desc.getColumn()))) {
@@ -159,6 +311,18 @@ public List generateColumnDescriptions() {
return descriptors;
}
+ @Override
+ protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecutionStatus status) {
+ status.setColumnDescriptions(generateColumnDescriptions());
+ }
+
+ @Override
+ public WorkerMessage createExecutionMessage() {
+ return new ExecuteForm(getId(), getFlatSubQueries().entrySet()
+ .stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getQuery())));
+ }
+
@Override
public List
](http://www.ingef.de/)
diff --git a/autodoc/src/main/java/com/bakdata/conquery/Constants.java b/autodoc/src/main/java/com/bakdata/conquery/Constants.java
index 36c3fe1fe4..005ad4a0b4 100644
--- a/autodoc/src/main/java/com/bakdata/conquery/Constants.java
+++ b/autodoc/src/main/java/com/bakdata/conquery/Constants.java
@@ -16,13 +16,13 @@
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
-import com.bakdata.conquery.apiv1.ExecutionStatus;
import com.bakdata.conquery.apiv1.FilterTemplate;
-import com.bakdata.conquery.apiv1.FullExecutionStatus;
import com.bakdata.conquery.apiv1.IdLabel;
import com.bakdata.conquery.apiv1.KeyValue;
import com.bakdata.conquery.apiv1.MetaDataPatch;
-import com.bakdata.conquery.apiv1.OverviewExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.ExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.FullExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus;
import com.bakdata.conquery.apiv1.frontend.FrontendRoot;
import com.bakdata.conquery.apiv1.frontend.FrontendValue;
import com.bakdata.conquery.apiv1.query.CQElement;
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java
index 7c1aeba082..6a05f227f8 100644
--- a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java
+++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java
@@ -1,6 +1,6 @@
package com.bakdata.conquery.apiv1;
-import java.net.URL;
+import java.net.URI;
import java.util.List;
import javax.validation.constraints.NotEmpty;
@@ -10,7 +10,7 @@
import com.bakdata.conquery.io.cps.CPSType;
import com.bakdata.conquery.io.jackson.serializer.NsIdRef;
import com.bakdata.conquery.io.storage.NamespaceStorage;
-import com.bakdata.conquery.models.config.SearchConfig;
+import com.bakdata.conquery.models.config.IndexConfig;
import com.bakdata.conquery.models.datasets.Dataset;
import com.bakdata.conquery.models.datasets.concepts.Searchable;
import com.bakdata.conquery.models.identifiable.IdentifiableImpl;
@@ -19,6 +19,7 @@
import com.bakdata.conquery.models.index.FrontendValueIndexKey;
import com.bakdata.conquery.models.index.IndexService;
import com.bakdata.conquery.models.index.search.SearchIndex;
+import com.bakdata.conquery.util.io.FileUtil;
import com.bakdata.conquery.util.search.TrieSearch;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
@@ -40,7 +41,7 @@
@ToString
@Slf4j
@CPSType(id = "CSV_TEMPLATE", base = SearchIndex.class)
-public class FilterTemplate extends IdentifiableImpl