From cf0ecd31f578d4675e6d1869fcdcc71d53faf5ab Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 4 Apr 2023 17:12:19 +0200 Subject: [PATCH 01/45] implements filter logic for empty values --- .../conquery/apiv1/FilterTemplate.java | 5 ++++- .../filters/specific/SelectFilter.java | 3 ++- .../models/index/FrontendValueIndex.java | 8 +++++-- .../models/index/FrontendValueIndexKey.java | 8 ++++--- .../filter/event/MultiSelectFilterNode.java | 21 ++++++++++++------- .../query/filter/event/SelectFilterNode.java | 18 ++++++++++++---- 6 files changed, 44 insertions(+), 19 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java index 6a05f227f8..a24ea2d384 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java @@ -74,6 +74,8 @@ public class FilterTemplate extends IdentifiableImpl implements S @NotEmpty private final String optionValue; + private final String emptyLabel; + private int minSuffixLength = 3; private boolean generateSuffixes = true; @@ -102,7 +104,8 @@ public List> getSearches(IndexConfig config, Namespace value, optionValue, isGenerateSuffixes() ? getMinSuffixLength() : Integer.MAX_VALUE, - config.getSearchSplitChars() + config.getSearchSplitChars(), + emptyLabel )); return List.of(search); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java index e8f06ecc24..79f22f7657 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java @@ -39,7 +39,7 @@ public abstract class SelectFilter extends SingleColumnFilter implements Searchable { /** - * user given mapping from the values in the CSVs to shown labels + * user given mapping from the values in the columns to shown labels */ protected BiMap labels = ImmutableBiMap.of(); @@ -65,6 +65,7 @@ public void configureFrontend(FrontendFilterConfiguration.Top f) throws ConceptC f.setCreatable(getSearchReferences().stream().noneMatch(Predicate.not(Searchable::isSearchDisabled))); f.setOptions(collectLabels()); + //TODO FK add empty label } @NotNull diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java index f9ea5345b9..69f272c206 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java @@ -1,14 +1,17 @@ package com.bakdata.conquery.models.index; +import java.util.List; import java.util.Map; import com.bakdata.conquery.apiv1.FilterTemplate; import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.models.query.FilterSearch; import com.bakdata.conquery.util.search.TrieSearch; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; @Slf4j +@ToString public class FrontendValueIndex extends TrieSearch implements Index { @@ -23,15 +26,16 @@ public class FrontendValueIndex extends TrieSearch implements Ind */ private final String optionValueTemplate; - public FrontendValueIndex(int suffixCutoff, String split, String valueTemplate, String optionValueTemplate) { + public FrontendValueIndex(int suffixCutoff, String split, String valueTemplate, String optionValueTemplate, String emptyLabel) { super(suffixCutoff, split); this.valueTemplate = valueTemplate; this.optionValueTemplate = optionValueTemplate; + addItem(new FrontendValue("", emptyLabel), List.of(emptyLabel, "")); } @Override public void put(String internalValue, Map templateToConcrete) { - FrontendValue feValue = new FrontendValue( + final FrontendValue feValue = new FrontendValue( internalValue, templateToConcrete.get(valueTemplate), templateToConcrete.get(optionValueTemplate) diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java index 5ca864c5ba..b4d91a293f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.index; import java.net.URI; -import java.net.URL; import java.util.List; import com.bakdata.conquery.apiv1.FilterTemplate; @@ -29,13 +28,16 @@ public class FrontendValueIndexKey extends AbstractIndexKey */ private final String optionValueTemplate; - public FrontendValueIndexKey(URI csv, String internalColumn, String valueTemplate, String optionValueTemplate, int suffixCutoff, String splitPattern) { + private final String emptyLabel; + + public FrontendValueIndexKey(URI csv, String internalColumn, String valueTemplate, String optionValueTemplate, int suffixCutoff, String splitPattern, String emptyLabel) { super(csv, internalColumn); this.suffixCutoff = suffixCutoff; this.splitPattern = splitPattern; this.valueTemplate = valueTemplate; this.optionValueTemplate = optionValueTemplate; + this.emptyLabel = emptyLabel; } @Override @@ -45,6 +47,6 @@ public List getExternalTemplates() { @Override public FrontendValueIndex createIndex() { - return new FrontendValueIndex(suffixCutoff, splitPattern, valueTemplate, optionValueTemplate); + return new FrontendValueIndex(suffixCutoff, splitPattern, valueTemplate, optionValueTemplate, emptyLabel); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java index 29e7001a57..445e528a1e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.query.filter.event; +import java.util.Arrays; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -15,20 +16,22 @@ import lombok.Getter; import lombok.Setter; import lombok.ToString; +import org.apache.logging.log4j.util.Strings; /** * Event is included when the value in column is one of many selected. */ -@ToString(callSuper = true, of = {"column"}) +@ToString(callSuper = true, of = "column") public class MultiSelectFilterNode extends EventFilterNode { - @NotNull @Getter @Setter private Column column; + private final boolean empty; + /** * Shared between all executing Threads to maximize utilization. */ @@ -39,6 +42,7 @@ public MultiSelectFilterNode(Column column, String[] filterValue) { super(filterValue); this.column = column; selectedValuesCache = new ConcurrentHashMap<>(); + empty = Arrays.stream(filterValue).anyMatch(Strings::isBlank); } @@ -55,13 +59,14 @@ public void nextBlock(Bucket bucket) { } private int[] findIds(Bucket bucket, String[] values) { - int[] selectedValues = new int[values.length]; + final int[] selectedValues = new int[values.length]; - StringStore type = (StringStore) bucket.getStore(getColumn()); + final StringStore type = (StringStore) bucket.getStore(getColumn()); for (int index = 0; index < values.length; index++) { - String select = values[index]; - int parsed = type.getId(select); + final String select = values[index]; + final int parsed = type.getId(select); + selectedValues[index] = parsed; } @@ -76,10 +81,10 @@ public boolean checkEvent(Bucket bucket, int event) { } if (!bucket.has(event, getColumn())) { - return false; + return empty; } - int stringToken = bucket.getString(event, getColumn()); + final int stringToken = bucket.getString(event, getColumn()); for (int selectedValue : selectedValues) { if (selectedValue == stringToken) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java index 181f9c398c..17ea23a913 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java @@ -12,6 +12,7 @@ import lombok.Getter; import lombok.Setter; import lombok.ToString; +import org.apache.logging.log4j.util.Strings; /** @@ -20,6 +21,7 @@ @ToString(callSuper = true, of = "column") public class SelectFilterNode extends EventFilterNode { + private final boolean empty; private int selectedId = -1; @NotNull @Getter @@ -29,28 +31,36 @@ public class SelectFilterNode extends EventFilterNode { public SelectFilterNode(Column column, String filterValue) { super(filterValue); this.column = column; + + empty = Strings.isBlank(filterValue); } @Override public void nextBlock(Bucket bucket) { - //you can then also skip the block if the id is -1 + // You can skip the block if the id is -1 selectedId = ((StringStore) bucket.getStore(getColumn())).getId(filterValue); } @Override public boolean checkEvent(Bucket bucket, int event) { - if (selectedId == -1 || !bucket.has(event, getColumn())) { + final boolean has = bucket.has(event, getColumn()); + + if(empty && !has){ + return true; + } + + if (selectedId == -1 || !has) { return false; } - int value = bucket.getString(event, getColumn()); + final int value = bucket.getString(event, getColumn()); return value == selectedId; } @Override public boolean isOfInterest(Bucket bucket) { - return ((StringStore) bucket.getStores()[getColumn().getPosition()]).getId(filterValue) != -1; + return empty || ((StringStore) bucket.getStores()[getColumn().getPosition()]).getId(filterValue) != -1; } @Override From 48bbaa9b13cd1eed45d76f7a33e2b441dd148ea9 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 11 Apr 2023 16:04:43 +0200 Subject: [PATCH 02/45] add emptyLabel as first entry in list endpoint. Also add emptyLabel es entry for searching --- .../conquery/apiv1/FilterTemplate.java | 2 + .../conquery/models/datasets/Column.java | 6 +++ .../models/datasets/concepts/Searchable.java | 2 + .../filters/specific/SelectFilter.java | 47 ++++++++++--------- .../models/index/FrontendValueIndex.java | 2 - .../resources/api/ConceptsProcessor.java | 13 +++-- 6 files changed, 44 insertions(+), 28 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java index a24ea2d384..436c98b81d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java @@ -108,6 +108,8 @@ public List> getSearches(IndexConfig config, Namespace emptyLabel )); + search.addItem(new FrontendValue("", getEmptyLabel()), List.of(getEmptyLabel())); + return List.of(search); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java index 1819a3f59c..215645bd68 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java @@ -54,6 +54,7 @@ public class Column extends Labeled implements NamespacedIdentifiable< private int minSuffixLength = 3; private boolean generateSuffixes; private boolean searchDisabled = false; + private String emptyLabel; @JsonIgnore @Getter(lazy = true) @@ -157,6 +158,8 @@ public List> getSearches(IndexConfig config, Namespace final TrieSearch search = new TrieSearch<>(suffixLength, config.getSearchSplitChars()); + search.addItem(new FrontendValue("", getEmptyLabel()), List.of(getEmptyLabel())); + storage.getAllImports().stream() .filter(imp -> imp.getTable().equals(getTable())) .flatMap(imp -> { @@ -168,6 +171,9 @@ public List> getSearches(IndexConfig config, Namespace .onClose(() -> log.debug("DONE processing values for {}", getId())) .forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); + + search.shrinkToFit(); + return List.of(search); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java index 1047c74a4c..84db1fcee7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java @@ -39,6 +39,8 @@ default List> getSearchReferences() { return List.of(this); } + String getEmptyLabel(); + /** * Parameter used in the construction of {@link com.bakdata.conquery.util.search.TrieSearch}, defining the shortest suffix to create. * Ignored if isGenerateSuffixes is true. diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java index 79f22f7657..9c15afd489 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java @@ -47,9 +47,10 @@ public abstract class SelectFilter extends SingleColumnFilter @NsIdRef @View.ApiManagerPersistence private FilterTemplate template; + private int searchMinSuffixLength = 3; + private boolean generateSearchSuffixes = true; - @JsonIgnore - public abstract String getFilterType(); + private String emptyLabel; @Override public EnumSet getAcceptedColumnTypes() { @@ -68,26 +69,8 @@ public void configureFrontend(FrontendFilterConfiguration.Top f) throws ConceptC //TODO FK add empty label } - @NotNull - protected List collectLabels() { - return labels.entrySet().stream() - .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())) - .collect(Collectors.toList()); - } - @JsonIgnore - @ValidationMethod(message = "Cannot use both labels and template.") - public boolean isNotUsingTemplateAndLabels() { - // Technically it's possible it just doesn't make much sense and would lead to Single-Point-of-Truth confusion. - if (getTemplate() == null && labels.isEmpty()) { - return true; - } - - return (getTemplate() == null) != labels.isEmpty(); - } - - private int searchMinSuffixLength = 3; - private boolean generateSearchSuffixes = true; + public abstract String getFilterType(); @Override public List> getSearchReferences() { @@ -106,6 +89,22 @@ public List> getSearchReferences() { return out; } + @NotNull + protected List collectLabels() { + return labels.entrySet().stream().map(entry -> new FrontendValue(entry.getKey(), entry.getValue())).collect(Collectors.toList()); + } + + @JsonIgnore + @ValidationMethod(message = "Cannot use both labels and template.") + public boolean isNotUsingTemplateAndLabels() { + // Technically it's possible it just doesn't make much sense and would lead to Single-Point-of-Truth confusion. + if (getTemplate() == null && labels.isEmpty()) { + return true; + } + + return (getTemplate() == null) != labels.isEmpty(); + } + @Override @JsonIgnore public boolean isGenerateSuffixes() { @@ -130,11 +129,15 @@ public boolean isSearchDisabled() { @Override public List> getSearches(IndexConfig config, NamespaceStorage storage) { - TrieSearch search = new TrieSearch<>(config.getSearchSuffixLength(), config.getSearchSplitChars()); + final TrieSearch search = new TrieSearch<>(config.getSearchSuffixLength(), config.getSearchSplitChars()); + + search.addItem(new FrontendValue("", getEmptyLabel()), List.of(getEmptyLabel())); + labels.entrySet() .stream() .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())) .forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); + search.shrinkToFit(); return List.of(search); diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java index 69f272c206..8411436581 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.index; -import java.util.List; import java.util.Map; import com.bakdata.conquery.apiv1.FilterTemplate; @@ -30,7 +29,6 @@ public FrontendValueIndex(int suffixCutoff, String split, String valueTemplate, super(suffixCutoff, split); this.valueTemplate = valueTemplate; this.optionValueTemplate = optionValueTemplate; - addItem(new FrontendValue("", emptyLabel), List.of(emptyLabel, "")); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index ea69f3d37e..6921468d3d 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -233,9 +233,15 @@ private Cursor listAllValues(Searchable searchable) { See: https://stackoverflow.com/questions/61114380/java-streams-buffering-huge-streams */ - final Iterator - iterators = - Iterators.concat(Iterators.transform(namespace.getFilterSearch().getSearchesFor(searchable).iterator(), TrieSearch::iterator)); + + final Iterator iterators = + Iterators.concat( + // We are always leading with the empty value. + Iterators.singletonIterator(new FrontendValue("", searchable.getEmptyLabel())), + Iterators.concat(Iterators.transform(namespace.getFilterSearch() + .getSearchesFor(searchable) + .iterator(), TrieSearch::iterator)) + ); // Use Set to accomplish distinct values final Set seen = new HashSet<>(); @@ -246,7 +252,6 @@ private Cursor listAllValues(Searchable searchable) { private long countAllValues(Searchable searchable) { final Namespace namespace = namespaces.get(searchable.getDataset().getId()); - return namespace.getFilterSearch().getTotal(searchable); } From 065a0f6b181d73a0bdfecbcf85799b00282adb19 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 12 Apr 2023 09:34:31 +0200 Subject: [PATCH 03/45] Implementation of emptyLabel as part Searchable. --- .../conquery/apiv1/FilterTemplate.java | 14 +- .../conquery/models/config/IndexConfig.java | 4 + .../conquery/models/datasets/Column.java | 8 +- .../models/datasets/concepts/Searchable.java | 4 +- .../filters/specific/SelectFilter.java | 18 +-- .../models/index/AbstractIndexKey.java | 1 - .../models/index/FrontendValueIndex.java | 3 +- .../models/index/FrontendValueIndexKey.java | 6 +- .../conquery/models/index/IndexKey.java | 2 +- .../conquery/models/index/IndexService.java | 26 ++-- .../conquery/models/index/MapIndexKey.java | 2 +- .../models/index/MapInternToExternMapper.java | 2 + .../models/jobs/UpdateFilterSearchJob.java | 9 +- .../resources/api/ConceptsProcessor.java | 8 +- .../conquery/util/search/TrieSearch.java | 131 +++++++++--------- .../tests/FilterAutocompleteTest.java | 3 +- .../tests/FilterResolutionTest.java | 2 +- .../models/index/IndexServiceTest.java | 43 +++--- 18 files changed, 134 insertions(+), 152 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java index 436c98b81d..03cda498ba 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.apiv1; import java.net.URI; -import java.util.List; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; @@ -74,8 +73,6 @@ public class FilterTemplate extends IdentifiableImpl implements S @NotEmpty private final String optionValue; - private final String emptyLabel; - private int minSuffixLength = 3; private boolean generateSuffixes = true; @@ -93,24 +90,21 @@ public boolean isSearchDisabled() { return false; } - public List> getSearches(IndexConfig config, NamespaceStorage storage) { + public TrieSearch createTrieSearch(IndexConfig config, NamespaceStorage storage) { final URI resolvedURI = FileUtil.getResolvedUri(config.getBaseUrl(), getFilePath()); log.trace("Resolved filter template reference url for search '{}': {}", this.getId(), resolvedURI); - FrontendValueIndex search = indexService.getIndex(new FrontendValueIndexKey( + final FrontendValueIndex search = indexService.getIndex(new FrontendValueIndexKey( resolvedURI, columnValue, value, optionValue, isGenerateSuffixes() ? getMinSuffixLength() : Integer.MAX_VALUE, - config.getSearchSplitChars(), - emptyLabel + config.getSearchSplitChars() )); - search.addItem(new FrontendValue("", getEmptyLabel()), List.of(getEmptyLabel())); - - return List.of(search); + return search; } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IndexConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IndexConfig.java index 9fc7a17102..c027587ff4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/IndexConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/IndexConfig.java @@ -5,6 +5,7 @@ import javax.annotation.Nullable; import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; import com.bakdata.conquery.models.index.IndexKey; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -30,6 +31,9 @@ public class IndexConfig { @Nullable private String searchSplitChars = "(),;.:\"'/"; + @NotNull + private String emptyLabel = "No Value"; + @JsonIgnore @ValidationMethod(message = "Specified baseUrl is not valid") public boolean isValidUrl() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java index 215645bd68..bea6fd2583 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java @@ -54,7 +54,6 @@ public class Column extends Labeled implements NamespacedIdentifiable< private int minSuffixLength = 3; private boolean generateSuffixes; private boolean searchDisabled = false; - private String emptyLabel; @JsonIgnore @Getter(lazy = true) @@ -152,14 +151,12 @@ private String computeDefaultDictionaryName(String importName) { @Override - public List> getSearches(IndexConfig config, NamespaceStorage storage) { + public TrieSearch createTrieSearch(IndexConfig config, NamespaceStorage storage) { final int suffixLength = isGenerateSuffixes() ? config.getSearchSuffixLength() : Integer.MAX_VALUE; final TrieSearch search = new TrieSearch<>(suffixLength, config.getSearchSplitChars()); - search.addItem(new FrontendValue("", getEmptyLabel()), List.of(getEmptyLabel())); - storage.getAllImports().stream() .filter(imp -> imp.getTable().equals(getTable())) .flatMap(imp -> { @@ -172,9 +169,8 @@ public List> getSearches(IndexConfig config, Namespace .forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); - search.shrinkToFit(); - return List.of(search); + return search; } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java index 84db1fcee7..ee8c13d5ab 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java @@ -26,7 +26,7 @@ public interface Searchable>> /** * All available {@link FrontendValue}s for searching in a {@link TrieSearch}. */ - List> getSearches(IndexConfig config, NamespaceStorage storage); + TrieSearch createTrieSearch(IndexConfig config, NamespaceStorage storage); /** * The actual Searchables to use, if there is potential for deduplication/pooling. @@ -39,8 +39,6 @@ default List> getSearchReferences() { return List.of(this); } - String getEmptyLabel(); - /** * Parameter used in the construction of {@link com.bakdata.conquery.util.search.TrieSearch}, defining the shortest suffix to create. * Ignored if isGenerateSuffixes is true. diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java index 9c15afd489..5a92cad3d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java @@ -50,8 +50,6 @@ public abstract class SelectFilter extends SingleColumnFilter private int searchMinSuffixLength = 3; private boolean generateSearchSuffixes = true; - private String emptyLabel; - @Override public EnumSet getAcceptedColumnTypes() { return EnumSet.of(MajorTypeId.STRING); @@ -66,7 +64,6 @@ public void configureFrontend(FrontendFilterConfiguration.Top f) throws ConceptC f.setCreatable(getSearchReferences().stream().noneMatch(Predicate.not(Searchable::isSearchDisabled))); f.setOptions(collectLabels()); - //TODO FK add empty label } @JsonIgnore @@ -91,7 +88,8 @@ public List> getSearchReferences() { @NotNull protected List collectLabels() { - return labels.entrySet().stream().map(entry -> new FrontendValue(entry.getKey(), entry.getValue())).collect(Collectors.toList()); + return labels.entrySet().stream() + .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())).collect(Collectors.toList()); } @JsonIgnore @@ -127,19 +125,13 @@ public boolean isSearchDisabled() { } @Override - public List> getSearches(IndexConfig config, NamespaceStorage storage) { + public TrieSearch createTrieSearch(IndexConfig config, NamespaceStorage storage) { final TrieSearch search = new TrieSearch<>(config.getSearchSuffixLength(), config.getSearchSplitChars()); - search.addItem(new FrontendValue("", getEmptyLabel()), List.of(getEmptyLabel())); - - labels.entrySet() - .stream() - .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())) - .forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); + collectLabels().forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); - search.shrinkToFit(); - return List.of(search); + return search; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java index dff8a9996f..df6b9cee45 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.index; import java.net.URI; -import java.net.URL; import lombok.Data; diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java index 8411436581..2951859937 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java @@ -25,7 +25,7 @@ public class FrontendValueIndex extends TrieSearch implements Ind */ private final String optionValueTemplate; - public FrontendValueIndex(int suffixCutoff, String split, String valueTemplate, String optionValueTemplate, String emptyLabel) { + public FrontendValueIndex(int suffixCutoff, String split, String valueTemplate, String optionValueTemplate) { super(suffixCutoff, split); this.valueTemplate = valueTemplate; this.optionValueTemplate = optionValueTemplate; @@ -54,6 +54,5 @@ public int size() { @Override public void finalizer() { - shrinkToFit(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java index b4d91a293f..6cc020c4d9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java @@ -28,16 +28,14 @@ public class FrontendValueIndexKey extends AbstractIndexKey */ private final String optionValueTemplate; - private final String emptyLabel; - public FrontendValueIndexKey(URI csv, String internalColumn, String valueTemplate, String optionValueTemplate, int suffixCutoff, String splitPattern, String emptyLabel) { + public FrontendValueIndexKey(URI csv, String internalColumn, String valueTemplate, String optionValueTemplate, int suffixCutoff, String splitPattern) { super(csv, internalColumn); this.suffixCutoff = suffixCutoff; this.splitPattern = splitPattern; this.valueTemplate = valueTemplate; this.optionValueTemplate = optionValueTemplate; - this.emptyLabel = emptyLabel; } @Override @@ -47,6 +45,6 @@ public List getExternalTemplates() { @Override public FrontendValueIndex createIndex() { - return new FrontendValueIndex(suffixCutoff, splitPattern, valueTemplate, optionValueTemplate, emptyLabel); + return new FrontendValueIndex(suffixCutoff, splitPattern, valueTemplate, optionValueTemplate); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java index fda0bedaf1..9c4b8bffd7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.index; import java.net.URI; -import java.net.URL; import java.util.List; /** @@ -26,4 +25,5 @@ public interface IndexKey>> { List getExternalTemplates(); I createIndex(); + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java index a8c3e9bd54..bff5f1498a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java @@ -34,9 +34,10 @@ public class IndexService implements Injectable { private final CsvParserSettings csvParserSettings; + private final LoadingCache, Index> mappings = CacheBuilder.newBuilder().build(new CacheLoader<>() { @Override - public Index load(@NotNull IndexKey key) throws Exception { + public Index load(@NotNull IndexKey key) throws Exception { log.info("Started to parse mapping {}", key); final Map emptyDefaults = computeEmptyDefaults(key); @@ -70,12 +71,9 @@ public Index load(@NotNull IndexKey key) throws Exception { int2ext.put(internalValue, externalValue); } catch (IllegalArgumentException e) { - log.warn( - "Skipping mapping '{}'->'{}' in row {}, because there was already a mapping", - internalValue, - externalValue, - csvParser.getContext().currentLine(), - (Exception) (log.isTraceEnabled() ? e : null) + log.warn("Skipping mapping '{}'->'{}' in row {}, because there was already a mapping", + internalValue, externalValue, csvParser.getContext().currentLine(), + (Exception) (log.isTraceEnabled() ? e : null) ); } } @@ -106,10 +104,10 @@ private Pair> computeInternalExternal(@NotNull Index final String internalValue = row.getString(key.getInternalColumn()); if (internalValue == null) { - log.trace( - "Could not create a mapping for row {} because the cell for the internal value was empty. Row: {}", - csvParser.getContext().currentLine(), - log.isTraceEnabled() ? StringUtils.join(row.toFieldMap()) : null + log.trace("Could not create a mapping for row {} because the cell for the internal value was empty. Row: {}", csvParser.getContext().currentLine(), + log.isTraceEnabled() + ? StringUtils.join(row.toFieldMap()) + : null ); return null; } @@ -127,11 +125,7 @@ private Map computeTemplates(StringSubstitutor substitutor, List return externalTemplates.stream() .distinct() - .collect(Collectors.toMap( - Functions.identity(), - value -> whitespaceMatcher.trimAndCollapseFrom(substitutor.replace(value), ' ') - ) - ); + .collect(Collectors.toMap(Functions.identity(), value -> whitespaceMatcher.trimAndCollapseFrom(substitutor.replace(value), ' '))); } private Map computeEmptyDefaults(IndexKey key) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java index cdaaad9619..2fb207bec6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.index; import java.net.URI; -import java.net.URL; import java.util.List; import lombok.EqualsAndHashCode; @@ -13,6 +12,7 @@ public class MapIndexKey extends AbstractIndexKey { private final String externalTemplate; + public MapIndexKey(URI csv, String internalColumn, String externalTemplate) { super(csv, internalColumn); this.externalTemplate = externalTemplate; diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java index 0ee8181697..74087d0f3e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java @@ -61,6 +61,8 @@ public class MapInternToExternMapper extends NamedImpl i @NotEmpty private final String externalTemplate; + private final String emptyLabel; + //Manager only @JsonIgnore diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java index 2b315e92bc..438667822d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java @@ -85,12 +85,15 @@ public void execute() throws Exception { log.info("BEGIN collecting entries for `{}`", searchable); try { - final List> values = searchable.getSearches(indexConfig, storage); + final TrieSearch search = searchable.createTrieSearch(indexConfig, storage); - for (TrieSearch search : values) { - synchronizedResult.put(searchable, search); + if(search.findExact(List.of(""), 1).isEmpty()){ + search.addItem(new FrontendValue("", indexConfig.getEmptyLabel()), List.of(indexConfig.getEmptyLabel())); } + search.shrinkToFit(); + synchronizedResult.put(searchable, search); + log.debug( "DONE collecting entries for `{}`, within {}", searchable, diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index 6921468d3d..d316295a46 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -27,6 +27,7 @@ import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.PreviewConfig; import com.bakdata.conquery.models.datasets.concepts.Concept; @@ -64,6 +65,8 @@ public class ConceptsProcessor { private final DatasetRegistry namespaces; private final Validator validator; + private final ConqueryConfig config; + private final LoadingCache, FrontendList> nodeCache = CacheBuilder.newBuilder().softValues().expireAfterWrite(10, TimeUnit.MINUTES).build(new CacheLoader<>() { @Override @@ -218,7 +221,7 @@ public AutoCompleteResult autocompleteTextFilter(Searchable searchable, Optio return new AutoCompleteResult(fullResult.subList(startIncl, Math.min(fullResult.size(), endExcl)), fullResult.size()); } catch (ExecutionException e) { - log.warn("Failed to search for \"{}\".", maybeText, log.isTraceEnabled() ? e : null); + log.warn("Failed to search for \"{}\".", maybeText, (Exception) (log.isTraceEnabled() ? e : null)); return new AutoCompleteResult(Collections.emptyList(), 0); } } @@ -233,11 +236,10 @@ private Cursor listAllValues(Searchable searchable) { See: https://stackoverflow.com/questions/61114380/java-streams-buffering-huge-streams */ - final Iterator iterators = Iterators.concat( // We are always leading with the empty value. - Iterators.singletonIterator(new FrontendValue("", searchable.getEmptyLabel())), + Iterators.singletonIterator(new FrontendValue("", config.getIndex().getEmptyLabel())), Iterators.concat(Iterators.transform(namespace.getFilterSearch() .getSearchesFor(searchable) .iterator(), TrieSearch::iterator)) diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java b/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java index 2877975714..d5c8622836 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java @@ -54,7 +54,10 @@ public class TrieSearch> { private final int suffixCutoff; private final Pattern splitPattern; - + /** + * Maps from keywords to associated items. + */ + private final PatriciaTrie> trie = new PatriciaTrie<>(); private boolean shrunk = false; private long size = -1; @@ -67,18 +70,41 @@ public TrieSearch(int suffixCutoff, String split) { splitPattern = Pattern.compile(String.format("[\\s%s]+", Pattern.quote(Objects.requireNonNullElse(split, "") + WHOLE_WORD_MARKER))); } - /** - * Maps from keywords to associated items. - */ - private final PatriciaTrie> trie = new PatriciaTrie<>(); + public List findItems(Collection keywords, int limit) { + final Object2DoubleMap itemWeights = new Object2DoubleAVLTreeMap<>(); - Stream suffixes(String word) { - return Stream.concat( - // We append a special character here marking original words as we want to favor them in weighing. - Stream.of(word + WHOLE_WORD_MARKER), - IntStream.range(1, Math.max(1, word.length() - suffixCutoff)) - .mapToObj(word::substring) - ); + // We are not guaranteed to have split keywords incoming, so we normalize them for searching + keywords = keywords.stream().flatMap(this::split).collect(Collectors.toSet()); + + for (String keyword : keywords) { + // Query trie for all items associated with extensions of keywords + final SortedMap> hits = trie.prefixMap(keyword); + + for (Map.Entry> entry : hits.entrySet()) { + + // calculate and update weights for all queried items + final String itemWord = entry.getKey(); + + final double weight = weightWord(keyword, itemWord); + + entry.getValue() + .forEach(item -> + { + // We combine hits multiplicative to favor items with multiple hits + final double currentWeight = itemWeights.getOrDefault(item, 1); + itemWeights.put(item, currentWeight * weight); + }); + } + } + + // Sort items according to their weight, then limit. + // Note that sorting is in ascending order, meaning lower-scores are better. + return itemWeights.object2DoubleEntrySet() + .stream() + .sorted(Comparator.comparingDouble(Object2DoubleMap.Entry::getDoubleValue)) + .limit(limit) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); } private Stream split(String keyword) { @@ -129,43 +155,6 @@ private boolean isOriginal(String itemWord) { return itemWord.endsWith(WHOLE_WORD_MARKER); } - public List findItems(Collection keywords, int limit) { - final Object2DoubleMap itemWeights = new Object2DoubleAVLTreeMap<>(); - - // We are not guaranteed to have split keywords incoming, so we normalize them for searching - keywords = keywords.stream().flatMap(this::split).collect(Collectors.toSet()); - - for (String keyword : keywords) { - // Query trie for all items associated with extensions of keywords - final SortedMap> hits = trie.prefixMap(keyword); - - for (Map.Entry> entry : hits.entrySet()) { - - // calculate and update weights for all queried items - final String itemWord = entry.getKey(); - - final double weight = weightWord(keyword, itemWord); - - entry.getValue() - .forEach(item -> - { - // We combine hits multiplicative to favor items with multiple hits - final double currentWeight = itemWeights.getOrDefault(item, 1); - itemWeights.put(item, currentWeight * weight); - }); - } - } - - // Sort items according to their weight, then limit. - // Note that sorting is in ascending order, meaning lower-scores are better. - return itemWeights.object2DoubleEntrySet() - .stream() - .sorted(Comparator.comparingDouble(Object2DoubleMap.Entry::getDoubleValue)) - .limit(limit) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - } - public List findExact(Collection keywords, int limit) { return keywords.stream() .flatMap(this::split) @@ -180,6 +169,24 @@ private Stream doGet(String kw) { return trie.getOrDefault(kw, Collections.emptyList()).stream(); } + public void addItem(T item, List keywords) { + // Associate item with all extracted keywords + keywords.stream() + .filter(Predicate.not(Strings::isNullOrEmpty)) + .flatMap(this::split) + .flatMap(this::suffixes) + .distinct() + .forEach(kw -> doPut(kw, item)); + } + + Stream suffixes(String word) { + return Stream.concat( + // We append a special character here marking original words as we want to favor them in weighing. + Stream.of(word + WHOLE_WORD_MARKER), + IntStream.range(1, Math.max(1, word.length() - suffixCutoff)) + .mapToObj(word::substring) + ); + } private void doPut(String kw, T item) { ensureWriteable(); @@ -195,16 +202,6 @@ private void ensureWriteable() { throw new IllegalStateException("Cannot alter a shrunk search."); } - public void addItem(T item, List keywords) { - // Associate item with all extracted keywords - keywords.stream() - .filter(Predicate.not(Strings::isNullOrEmpty)) - .flatMap(this::split) - .flatMap(this::suffixes) - .distinct() - .forEach(kw -> doPut(kw, item)); - } - public Collection listItems() { //TODO this a pretty dangerous operation, I'd rather see a session based iterator instead return trie.values().stream() @@ -214,14 +211,6 @@ public Collection listItems() { .collect(Collectors.toList()); } - public long calculateSize() { - if (size != -1) { - return size; - } - - return trie.values().stream().distinct().count(); - } - /** * Since growth of ArrayList might be excessive, we can shrink the internal lists to only required size instead. * @@ -238,6 +227,14 @@ public void shrinkToFit() { shrunk = true; } + public long calculateSize() { + if (size != -1) { + return size; + } + + return trie.values().stream().distinct().count(); + } + public void logStats() { final IntSummaryStatistics statistics = trie.values() diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java index a379e4b375..efd5af334d 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java @@ -120,7 +120,8 @@ public void execute(StandaloneSupport conquery) throws Exception { ), MediaType.APPLICATION_JSON_TYPE)); final ConceptsProcessor.AutoCompleteResult resolvedFromCsv = fromCsvResponse.readEntity(ConceptsProcessor.AutoCompleteResult.class); - assertThat(resolvedFromCsv.values().stream().map(FrontendValue::getValue)).containsExactly("a", "aaa", "aab", "baaa"); + assertThat(resolvedFromCsv.values().stream().map(FrontendValue::getValue)) + .containsExactly("a", "aaa", "aab", "baaa", "" /* `No V*a*lue` :^) */); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterResolutionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterResolutionTest.java index a5602a7cc0..a2e9d46975 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterResolutionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterResolutionTest.java @@ -79,7 +79,7 @@ public void execute(StandaloneSupport conquery) throws Exception { final IndexService indexService = new IndexService(conquery.getConfig().getCsv().createCsvParserSettings()); - filter.setTemplate(new FilterTemplate(conquery.getDataset(), "test", tmpCSv.toUri(), "HEADER", "", "", 2, true, indexService)); + filter.setTemplate(new FilterTemplate(conquery.getDataset(), "test", tmpCSv.toUri(), "HEADER", "", "", 2, true, indexService)); final URI matchingStatsUri = HierarchyHelper.hierarchicalPath(conquery.defaultAdminURIBuilder() , AdminDatasetResource.class, "updateMatchingStats") diff --git a/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java b/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java index da2ac24c67..a1309e05e9 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java @@ -29,11 +29,10 @@ @Slf4j public class IndexServiceTest { - private final IndexService indexService = new IndexService(new CsvParserSettings()); private final static Dataset DATASET = new Dataset("dataset"); private final static ConqueryConfig CONFIG = new ConqueryConfig(); - private final static ClientAndServer REF_SERVER = ClientAndServer.startClientAndServer(); + private final IndexService indexService = new IndexService(new CsvParserSettings()); @BeforeAll @SneakyThrows @@ -63,21 +62,24 @@ void testLoading() throws NoSuchFieldException, IllegalAccessException, URISynta "test1", new URI("classpath:/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv"), "internal", - "{{external}}" + "{{external}}", + "no value" ); final MapInternToExternMapper mapperUrlAbsolute = new MapInternToExternMapper( "testUrlAbsolute", new URI(String.format("http://localhost:%d/mapping.csv", REF_SERVER.getPort())), "internal", - "{{external}}" + "{{external}}", + "no value" ); final MapInternToExternMapper mapperUrlRelative = new MapInternToExternMapper( "testUrlRelative", new URI("./mapping.csv"), "internal", - "{{external}}" + "{{external}}", + "no value" ); @@ -100,6 +102,20 @@ void testLoading() throws NoSuchFieldException, IllegalAccessException, URISynta } + private static void injectComponents(MapInternToExternMapper mapInternToExternMapper, IndexService indexService, ConqueryConfig config) + throws NoSuchFieldException, IllegalAccessException { + + final Field indexServiceField = MapInternToExternMapper.class.getDeclaredField(MapInternToExternMapper.Fields.mapIndex); + indexServiceField.setAccessible(true); + indexServiceField.set(mapInternToExternMapper, indexService); + + final Field configField = MapInternToExternMapper.class.getDeclaredField(MapInternToExternMapper.Fields.config); + configField.setAccessible(true); + configField.set(mapInternToExternMapper, config); + + mapInternToExternMapper.setDataset(DATASET); + } + @Test @Order(2) void testEvictOnMapper() @@ -109,7 +125,8 @@ void testEvictOnMapper() "test1", new URI("classpath:/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv"), "internal", - "{{external}}" + "{{external}}", + "no value" ); injectComponents(mapInternToExternMapper, indexService, CONFIG); @@ -135,18 +152,4 @@ void testEvictOnMapper() .isNotSameAs(mappingAfterEvict); } - private static void injectComponents(MapInternToExternMapper mapInternToExternMapper, IndexService indexService, ConqueryConfig config) - throws NoSuchFieldException, IllegalAccessException { - - final Field indexServiceField = MapInternToExternMapper.class.getDeclaredField(MapInternToExternMapper.Fields.mapIndex); - indexServiceField.setAccessible(true); - indexServiceField.set(mapInternToExternMapper, indexService); - - final Field configField = MapInternToExternMapper.class.getDeclaredField(MapInternToExternMapper.Fields.config); - configField.setAccessible(true); - configField.set(mapInternToExternMapper, config); - - mapInternToExternMapper.setDataset(DATASET); - } - } From 19b823ec6c12208dbc1e3e37d02969345a8d66d3 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 12 Apr 2023 09:43:16 +0200 Subject: [PATCH 04/45] adds test for listing all values --- .../tests/FilterAutocompleteTest.java | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java index efd5af334d..2882416181 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java @@ -50,26 +50,25 @@ public class FilterAutocompleteTest extends IntegrationTest.Simple implements Pr @Override public void execute(StandaloneSupport conquery) throws Exception { //read test specification - String - testJson = + final String testJson = In.resource("/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY.test.json") .withUTF8() .readAll(); - DatasetId dataset = conquery.getDataset().getId(); + final DatasetId dataset = conquery.getDataset().getId(); - ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson); + final ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson); ValidatorHelper.failOnError(log, conquery.getValidator().validate(test)); test.importRequiredData(conquery); - CSVConfig csvConf = conquery.getConfig().getCsv(); + final CSVConfig csvConf = conquery.getConfig().getCsv(); conquery.waitUntilWorkDone(); - Concept concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next(); - Connector connector = concept.getConnectors().iterator().next(); - SelectFilter filter = (SelectFilter) connector.getFilters().iterator().next(); + final Concept concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next(); + final Connector connector = concept.getConnectors().iterator().next(); + final SelectFilter filter = (SelectFilter) connector.getFilters().iterator().next(); // Copy search csv from resources to tmp folder. final Path tmpCSv = Files.createTempFile("conquery_search", "csv"); @@ -141,5 +140,21 @@ public void execute(StandaloneSupport conquery) throws Exception { assertThat(resolvedFromValues.values().stream().map(FrontendValue::getValue)) .containsExactly("f", "fm"); } + + + // Data starting with a is in reference csv + { + final Response fromCsvResponse = conquery.getClient().target(autocompleteUri) + .request(MediaType.APPLICATION_JSON_TYPE) + .post(Entity.entity(new FilterResource.AutocompleteRequest( + Optional.of(""), + OptionalInt.empty(), + OptionalInt.empty() + ), MediaType.APPLICATION_JSON_TYPE)); + + final ConceptsProcessor.AutoCompleteResult resolvedFromCsv = fromCsvResponse.readEntity(ConceptsProcessor.AutoCompleteResult.class); + assertThat(resolvedFromCsv.values().stream().map(FrontendValue::getValue)) + .containsExactly("", "aaa", "a", "baaa", "aab", "b", "f", "fm", "m", "mf"); + } } } \ No newline at end of file From 89e1be64e29f3632af2f44fd8f8315336baae7f4 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 12 Apr 2023 10:02:21 +0200 Subject: [PATCH 05/45] use isEmpty not isBlank for empty test --- .../models/query/filter/event/MultiSelectFilterNode.java | 2 +- .../conquery/models/query/filter/event/SelectFilterNode.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java index 445e528a1e..42e2601419 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java @@ -42,7 +42,7 @@ public MultiSelectFilterNode(Column column, String[] filterValue) { super(filterValue); this.column = column; selectedValuesCache = new ConcurrentHashMap<>(); - empty = Arrays.stream(filterValue).anyMatch(Strings::isBlank); + empty = Arrays.stream(filterValue).anyMatch(Strings::isEmpty); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java index 17ea23a913..2af64b1942 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java @@ -32,7 +32,7 @@ public SelectFilterNode(Column column, String filterValue) { super(filterValue); this.column = column; - empty = Strings.isBlank(filterValue); + empty = Strings.isEmpty(filterValue); } @Override From 4566cd583c857cf93e28fe472b879f2ac9bebf49 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 12 Apr 2023 16:14:40 +0200 Subject: [PATCH 06/45] remove unused emptyLabel field --- .../bakdata/conquery/models/index/MapInternToExternMapper.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java index 74087d0f3e..0ee8181697 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java @@ -61,8 +61,6 @@ public class MapInternToExternMapper extends NamedImpl i @NotEmpty private final String externalTemplate; - private final String emptyLabel; - //Manager only @JsonIgnore From f6c1484b0c47af032c5e63226cd01fea26fd7594 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 12 Apr 2023 16:17:25 +0200 Subject: [PATCH 07/45] remove unused parameter --- .../models/index/IndexServiceTest.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java b/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java index a1309e05e9..de70f28240 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java @@ -25,13 +25,13 @@ import org.mockserver.model.HttpResponse; import org.mockserver.model.MediaType; -@TestMethodOrder(value = MethodOrderer.OrderAnnotation.class) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) @Slf4j public class IndexServiceTest { - private final static Dataset DATASET = new Dataset("dataset"); - private final static ConqueryConfig CONFIG = new ConqueryConfig(); - private final static ClientAndServer REF_SERVER = ClientAndServer.startClientAndServer(); + private static final Dataset DATASET = new Dataset("dataset"); + private static final ConqueryConfig CONFIG = new ConqueryConfig(); + private static final ClientAndServer REF_SERVER = ClientAndServer.startClientAndServer(); private final IndexService indexService = new IndexService(new CsvParserSettings()); @BeforeAll @@ -62,24 +62,21 @@ void testLoading() throws NoSuchFieldException, IllegalAccessException, URISynta "test1", new URI("classpath:/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv"), "internal", - "{{external}}", - "no value" + "{{external}}" ); final MapInternToExternMapper mapperUrlAbsolute = new MapInternToExternMapper( "testUrlAbsolute", new URI(String.format("http://localhost:%d/mapping.csv", REF_SERVER.getPort())), "internal", - "{{external}}", - "no value" + "{{external}}" ); final MapInternToExternMapper mapperUrlRelative = new MapInternToExternMapper( "testUrlRelative", new URI("./mapping.csv"), "internal", - "{{external}}", - "no value" + "{{external}}" ); @@ -125,8 +122,7 @@ void testEvictOnMapper() "test1", new URI("classpath:/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv"), "internal", - "{{external}}", - "no value" + "{{external}}" ); injectComponents(mapInternToExternMapper, indexService, CONFIG); @@ -136,7 +132,7 @@ void testEvictOnMapper() assertThat(mapInternToExternMapper.external("int1")).as("Internal Value").isEqualTo("hello"); - MapIndex mappingBeforeEvict = mapInternToExternMapper.getInt2ext(); + final MapIndex mappingBeforeEvict = mapInternToExternMapper.getInt2ext(); indexService.evictCache(); @@ -145,7 +141,7 @@ void testEvictOnMapper() mapInternToExternMapper.init(); - MapIndex mappingAfterEvict = mapInternToExternMapper.getInt2ext(); + final MapIndex mappingAfterEvict = mapInternToExternMapper.getInt2ext(); // Check that the mapping reinitialized assertThat(mappingBeforeEvict).as("Mapping before and after eviction") From 7a0817dceb94c6934da23d00c9292cded7d40e72 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 12 Apr 2023 17:46:51 +0200 Subject: [PATCH 08/45] fixes writing to closed search when it is deduplicated --- .../bakdata/conquery/models/jobs/UpdateFilterSearchJob.java | 4 ++-- .../java/com/bakdata/conquery/util/search/TrieSearch.java | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java index 438667822d..2ec42deefd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java @@ -87,11 +87,11 @@ public void execute() throws Exception { try { final TrieSearch search = searchable.createTrieSearch(indexConfig, storage); - if(search.findExact(List.of(""), 1).isEmpty()){ + if(search.isWriteable() && search.findExact(List.of(""), 1).isEmpty()){ search.addItem(new FrontendValue("", indexConfig.getEmptyLabel()), List.of(indexConfig.getEmptyLabel())); + search.shrinkToFit(); } - search.shrinkToFit(); synchronizedResult.put(searchable, search); log.debug( diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java b/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java index d5c8622836..a10f3453d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java @@ -196,7 +196,7 @@ private void doPut(String kw, T item) { } private void ensureWriteable() { - if (!shrunk) { + if (isWriteable()) { return; } throw new IllegalStateException("Cannot alter a shrunk search."); @@ -266,4 +266,8 @@ public Iterator iterator() { seen::add ); } + + public boolean isWriteable() { + return !shrunk; + } } From d7e444afb526d5d9eedffe463f4f459aea81db9f Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 19 Apr 2023 12:57:42 +0200 Subject: [PATCH 09/45] adds logging around FilterSearch --- .../java/com/bakdata/conquery/models/datasets/Column.java | 7 ------- .../conquery/models/jobs/UpdateFilterSearchJob.java | 7 ++++--- .../com/bakdata/conquery/models/query/FilterSearch.java | 8 ++++++-- .../bakdata/conquery/resources/api/ConceptsProcessor.java | 8 +++++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java index bea6fd2583..f7303d091b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.datasets; -import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -172,10 +171,4 @@ public TrieSearch createTrieSearch(IndexConfig config, NamespaceS return search; } - - @Override - public List> getSearchReferences() { - return List.of(this); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java index 2ec42deefd..2821fe0577 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java @@ -82,7 +82,7 @@ public void execute() throws Exception { final StopWatch watch = StopWatch.createStarted(); - log.info("BEGIN collecting entries for `{}`", searchable); + log.info("BEGIN collecting entries for `{}`", searchable.getId()); try { final TrieSearch search = searchable.createTrieSearch(indexConfig, storage); @@ -95,8 +95,9 @@ public void execute() throws Exception { synchronizedResult.put(searchable, search); log.debug( - "DONE collecting entries for `{}`, within {}", - searchable, + "DONE collecting {} entries for `{}`, within {}", + search.calculateSize(), + searchable.getId(), Duration.ofMillis(watch.getTime()) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java index 63850ff7e2..b67a908a2c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java @@ -62,8 +62,12 @@ public static List extractKeywords(FrontendValue value) { /** * For a {@link SelectFilter} collect all relevant {@link TrieSearch}. */ - public List> getSearchesFor(Searchable searchable) { - return searchable.getSearchReferences().stream() + public final List> getSearchesFor(Searchable searchable) { + final List> references = searchable.getSearchReferences(); + + log.debug("Got {} as searchables for {}", references.stream().map(Searchable::getId).collect(Collectors.toList()), searchable.getId()); + + return references.stream() .map(searchCache::get) .filter(Objects::nonNull) .collect(Collectors.toList()); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index a049a16437..7f15a02e81 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -25,7 +25,6 @@ import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.permissions.Ability; -import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.PreviewConfig; @@ -235,12 +234,15 @@ private Cursor listAllValues(Searchable searchable) { See: https://stackoverflow.com/questions/61114380/java-streams-buffering-huge-streams */ + List> searches = namespace.getFilterSearch().getSearchesFor(searchable); + + log.debug(""); + final Iterator iterators = Iterators.concat( // We are always leading with the empty value. Iterators.singletonIterator(new FrontendValue("", config.getIndex().getEmptyLabel())), - Iterators.concat(Iterators.transform(namespace.getFilterSearch() - .getSearchesFor(searchable) + Iterators.concat(Iterators.transform(searches .iterator(), TrieSearch::iterator)) ); From a905ff153a82bb93794c4af27f28b4b9cd1d52bc Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 19 Apr 2023 14:54:15 +0200 Subject: [PATCH 10/45] log collectLabels content --- .../datasets/concepts/filters/specific/SelectFilter.java | 5 ++++- .../bakdata/conquery/resources/api/ConceptsProcessor.java | 7 ++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java index 5a92cad3d6..9b4eadb002 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java @@ -89,7 +89,8 @@ public List> getSearchReferences() { @NotNull protected List collectLabels() { return labels.entrySet().stream() - .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())).collect(Collectors.toList()); + .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); } @JsonIgnore @@ -129,6 +130,8 @@ public TrieSearch createTrieSearch(IndexConfig config, NamespaceS final TrieSearch search = new TrieSearch<>(config.getSearchSuffixLength(), config.getSearchSplitChars()); + log.debug("Labels for {}: `{}`", getId(), collectLabels().stream().map(FrontendValue::toString).collect(Collectors.toList())); + collectLabels().forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index 7f15a02e81..b2e13edbfd 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -234,16 +234,13 @@ private Cursor listAllValues(Searchable searchable) { See: https://stackoverflow.com/questions/61114380/java-streams-buffering-huge-streams */ - List> searches = namespace.getFilterSearch().getSearchesFor(searchable); - - log.debug(""); + final List> searches = namespace.getFilterSearch().getSearchesFor(searchable); final Iterator iterators = Iterators.concat( // We are always leading with the empty value. Iterators.singletonIterator(new FrontendValue("", config.getIndex().getEmptyLabel())), - Iterators.concat(Iterators.transform(searches - .iterator(), TrieSearch::iterator)) + Iterators.concat(Iterators.transform(searches.iterator(), TrieSearch::iterator)) ); // Use Set to accomplish distinct values From b42ed7d9f9c019fa1808de1864af25b0d8432f50 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 20 Apr 2023 09:51:25 +0200 Subject: [PATCH 11/45] fixes a bug where labels were empty --- .../filters/specific/BigMultiSelectFilter.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java index 7fa0bdb7bb..f17e80851a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java @@ -1,10 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; -import java.util.Collections; -import java.util.List; - import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; -import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.query.filter.event.MultiSelectFilterNode; @@ -12,7 +8,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; -import org.jetbrains.annotations.NotNull; /** * This filter represents a select in the front end. This means that the user can select one or more values from a list of values. @@ -35,10 +30,4 @@ public String getFilterType() { public FilterNode createFilterNode(String[] value) { return new MultiSelectFilterNode(getColumn(), value); } - - @NotNull - protected List collectLabels() { - // Frontend expects no Labels when encountering BIG_MULTI_SELECT - return Collections.emptyList(); - } } From 3e431ca3d3a4eeca8050a408c816c38e02c708ab Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 20 Apr 2023 09:54:03 +0200 Subject: [PATCH 12/45] reduces verbose logging to trace --- .../datasets/concepts/filters/specific/SelectFilter.java | 4 +++- .../java/com/bakdata/conquery/models/query/FilterSearch.java | 4 +++- .../com/bakdata/conquery/resources/api/ConceptsProcessor.java | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java index 9b4eadb002..8a547e5308 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java @@ -130,7 +130,9 @@ public TrieSearch createTrieSearch(IndexConfig config, NamespaceS final TrieSearch search = new TrieSearch<>(config.getSearchSuffixLength(), config.getSearchSplitChars()); - log.debug("Labels for {}: `{}`", getId(), collectLabels().stream().map(FrontendValue::toString).collect(Collectors.toList())); + if(log.isTraceEnabled()) { + log.trace("Labels for {}: `{}`", getId(), collectLabels().stream().map(FrontendValue::toString).collect(Collectors.toList())); + } collectLabels().forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java index b67a908a2c..fabf329c07 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java @@ -65,7 +65,9 @@ public static List extractKeywords(FrontendValue value) { public final List> getSearchesFor(Searchable searchable) { final List> references = searchable.getSearchReferences(); - log.debug("Got {} as searchables for {}", references.stream().map(Searchable::getId).collect(Collectors.toList()), searchable.getId()); + if(log.isTraceEnabled()) { + log.trace("Got {} as searchables for {}", references.stream().map(Searchable::getId).collect(Collectors.toList()), searchable.getId()); + } return references.stream() .map(searchCache::get) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index b2e13edbfd..c59be20ec6 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -98,7 +98,7 @@ public List load(Pair, String> filterAndSearch) { private final LoadingCache, CursorAndLength> listResults = CacheBuilder.newBuilder().softValues().build(new CacheLoader<>() { @Override public CursorAndLength load(Searchable searchable) { - log.debug("Creating cursor for `{}`", searchable.getId()); + log.trace("Creating cursor for `{}`", searchable.getId()); return new CursorAndLength(listAllValues(searchable), countAllValues(searchable)); } From f5e63c287fe0ad6c083dbcd837eb82f793aef5f1 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 2 May 2023 11:33:37 +0200 Subject: [PATCH 13/45] Support more time stratified infos --- frontend/src/js/api/types.ts | 4 ++-- frontend/src/js/entity-history/EntityCard.tsx | 24 ++++++++++++++----- .../js/entity-history/timeline/YearHead.tsx | 22 +++++++++++++---- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/frontend/src/js/api/types.ts b/frontend/src/js/api/types.ts index c0f66fb72a..55d19c2008 100644 --- a/frontend/src/js/api/types.ts +++ b/frontend/src/js/api/types.ts @@ -553,7 +553,7 @@ export interface TimeStratifiedInfoQuarter { export interface TimeStratifiedInfoYear { year: number; values: { - [label: string]: number; + [label: string]: number | string[]; }; quarters: TimeStratifiedInfoQuarter[]; } @@ -562,7 +562,7 @@ export interface TimeStratifiedInfo { label: string; description: string | null; totals: { - [label: string]: number; + [label: string]: number | string[]; }; columns: { label: string; // Matches `label` with `year.values` and `year.quarters[].values` diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index c6ff033076..f89275df8b 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -11,8 +11,8 @@ import { getColumnType } from "./timeline/util"; const Container = styled("div")` display: grid; - grid-template-columns: 1.618fr 1fr; - gap: 30px; + grid-template-columns: 1fr 1.618fr; + gap: 10px; padding: 20px; background-color: ${({ theme }) => theme.col.bg}; border-radius: ${({ theme }) => theme.borderRadius}; @@ -24,14 +24,22 @@ const Centered = styled("div")` align-items: flex-start; `; +const Col = styled("div")` + display: flex; + flex-direction: column; + gap: 10px; +`; + const Grid = styled("div")` - display: inline-grid; + display: grid; gap: 0 20px; grid-template-columns: auto auto; `; + const Label = styled("div")` font-size: ${({ theme }) => theme.font.xs}; `; + const Value = styled("div")` font-size: ${({ theme }) => theme.font.sm}; font-weight: 400; @@ -48,7 +56,7 @@ const TimeStratifiedInfos = ({ ); return ( -
+ {timeStratifiedInfos.map((timeStratifiedInfo) => { return ( @@ -64,7 +72,11 @@ const TimeStratifiedInfos = ({ if (!exists(value)) return <>; const valueFormatted = - columnType === "MONEY" ? Math.round(value) : value; + typeof value === "number" + ? Math.round(value) + : value instanceof Array + ? value.join(", ") + : value; return ( @@ -79,7 +91,7 @@ const TimeStratifiedInfos = ({ ); })} -
+ ); }; diff --git a/frontend/src/js/entity-history/timeline/YearHead.tsx b/frontend/src/js/entity-history/timeline/YearHead.tsx index bc95a8942e..94ad76ebd0 100644 --- a/frontend/src/js/entity-history/timeline/YearHead.tsx +++ b/frontend/src/js/entity-history/timeline/YearHead.tsx @@ -32,6 +32,12 @@ const StickyWrap = styled("div")` } `; +const Col = styled("div")` + display: flex; + flex-direction: column; + gap: 6px; +`; + const Grid = styled("div")` display: grid; grid-template-columns: auto 45px; @@ -41,8 +47,12 @@ const Grid = styled("div")` const Value = styled("div")` font-size: ${({ theme }) => theme.font.tiny}; font-weight: 400; - white-space: nowrap; justify-self: end; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + width: 100%; + text-align: right; `; const Label = styled("div")` @@ -81,7 +91,7 @@ const TimeStratifiedInfos = ({ ); return ( - <> + {infos.map(({ info, yearInfo }) => { return ( @@ -94,7 +104,11 @@ const TimeStratifiedInfos = ({ .map(([label, value]) => { const columnType = getColumnType(info, label); const valueFormatted = - columnType === "MONEY" ? Math.round(value) : value; + typeof value === "number" + ? Math.round(value) + : value instanceof Array + ? value.join(", ") + : value; return ( @@ -109,7 +123,7 @@ const TimeStratifiedInfos = ({ ); })} - + ); }; From 819c92aef6d99dc030aa6e6be76d03eb93366630 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 2 May 2023 13:34:24 +0200 Subject: [PATCH 14/45] Start iterating with charts --- frontend/package.json | 2 + frontend/src/js/entity-history/EntityCard.tsx | 132 ++++++++++++++++-- frontend/yarn.lock | 17 +++ 3 files changed, 143 insertions(+), 8 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 6442e257c9..5afc43037c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -40,6 +40,7 @@ "@vitejs/plugin-react": "^3.1.0", "axios": "^1.3.4", "chance": "^1.1.11", + "chart.js": "^4.3.0", "compression": "^1.7.4", "date-fns": "^2.28.0", "downshift": "^7.4.1", @@ -53,6 +54,7 @@ "mustache": "^4.2.0", "nodemon": "^2.0.21", "react": "^18.1.0", + "react-chartjs-2": "^5.2.0", "react-datepicker": "^4.11.0", "react-dnd": "^16.0.1", "react-dnd-html5-backend": "^16.0.1", diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index f89275df8b..23ef7b7f8b 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -1,5 +1,15 @@ +import { useTheme } from "@emotion/react"; import styled from "@emotion/styled"; -import { Fragment } from "react"; +import { + Chart as ChartJS, + CategoryScale, + LinearScale, + BarElement, + Tooltip, + ChartOptions, +} from "chart.js"; +import { Fragment, useMemo } from "react"; +import { Bar } from "react-chartjs-2"; import { useSelector } from "react-redux"; import { EntityInfo, TimeStratifiedInfo } from "../api/types"; @@ -11,7 +21,7 @@ import { getColumnType } from "./timeline/util"; const Container = styled("div")` display: grid; - grid-template-columns: 1fr 1.618fr; + grid-template-columns: 1fr 1fr; gap: 10px; padding: 20px; background-color: ${({ theme }) => theme.col.bg}; @@ -30,6 +40,11 @@ const Col = styled("div")` gap: 10px; `; +const ChartContainer = styled("div")` + height: 200px; + width: 100%; +`; + const Grid = styled("div")` display: grid; gap: 0 20px; @@ -46,17 +61,37 @@ const Value = styled("div")` justify-self: end; `; -const TimeStratifiedInfos = ({ +ChartJS.register( + CategoryScale, + LinearScale, + BarElement, + // Title, + Tooltip, + // Legend, +); + +function hexToRgbA(hex: string) { + let c: any; + if (/^#([A-Fa-f0-9]{3}){1,2}$/.test(hex)) { + c = hex.substring(1).split(""); + if (c.length === 3) { + c = [c[0], c[0], c[1], c[1], c[2], c[2]]; + } + c = "0x" + c.join(""); + return [(c >> 16) & 255, (c >> 8) & 255, c & 255].join(","); + } + throw new Error("Bad Hex"); +} + +const Table = ({ timeStratifiedInfos, + currencyUnit, }: { timeStratifiedInfos: TimeStratifiedInfo[]; + currencyUnit: string; }) => { - const currencyUnit = useSelector( - (state) => state.startup.config.currency.unit, - ); - return ( - + <> {timeStratifiedInfos.map((timeStratifiedInfo) => { return ( @@ -91,6 +126,87 @@ const TimeStratifiedInfos = ({ ); })} + + ); +}; + +const TimeStratifiedInfos = ({ + timeStratifiedInfos, +}: { + timeStratifiedInfos: TimeStratifiedInfo[]; +}) => { + const currencyUnit = useSelector( + (state) => state.startup.config.currency.unit, + ); + + const theme = useTheme(); + const datasets = useMemo(() => { + const sortedYears = [...timeStratifiedInfos[0].years].sort( + (a, b) => b.year - a.year, + ); + + return sortedYears.map((year, i) => { + return { + label: year.year.toString(), + data: Object.values(year.values), + backgroundColor: `rgba(${hexToRgbA(theme.col.blueGrayDark)}, ${1 / i})`, + }; + }); + }, [theme, timeStratifiedInfos]); + + const entries = Object.entries(timeStratifiedInfos[0].totals); + const labels = entries.map(([key]) => key); + // const values = entries.map(([, value]) => value); + + const data = { + labels, + datasets, + }; + + const options: ChartOptions<"bar"> = useMemo(() => { + return { + // indexAxis: "y" as const, + // legend: { + // position: "right" as const, + // }, + plugins: { + title: { + display: true, + text: timeStratifiedInfos[0].label, + }, + subtitle: { + text: "ololol", + }, + }, + responsive: true, + interaction: { + mode: "index" as const, + intersect: false, + }, + datasets: {}, + layout: { + padding: 0, + }, + // scales: { + // x: { + // stacked: true, + // }, + // y: { + // stacked: true, + // }, + // }, + }; + }, [timeStratifiedInfos]); + + return ( + + + + + {/* */} ); }; diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 1b75b6cd7b..792f7e87d7 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -1789,6 +1789,11 @@ resolved "https://registry.yarnpkg.com/@juggle/resize-observer/-/resize-observer-3.4.0.tgz#08d6c5e20cf7e4cc02fd181c4b0c225cd31dbb60" integrity sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA== +"@kurkle/color@^0.3.0": + version "0.3.2" + resolved "https://registry.yarnpkg.com/@kurkle/color/-/color-0.3.2.tgz#5acd38242e8bde4f9986e7913c8fdf49d3aa199f" + integrity sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw== + "@mdx-js/react@^2.1.5": version "2.3.0" resolved "https://registry.yarnpkg.com/@mdx-js/react/-/react-2.3.0.tgz#4208bd6d70f0d0831def28ef28c26149b03180b3" @@ -4137,6 +4142,13 @@ character-entities@^2.0.0: resolved "https://registry.yarnpkg.com/character-entities/-/character-entities-2.0.2.tgz#2d09c2e72cd9523076ccb21157dff66ad43fcc22" integrity sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ== +chart.js@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-4.3.0.tgz#ac363030ab3fec572850d2d872956f32a46326a1" + integrity sha512-ynG0E79xGfMaV2xAHdbhwiPLczxnNNnasrmPEXriXsPJGjmhOBYzFVEsB65w2qMDz+CaBJJuJD0inE/ab/h36g== + dependencies: + "@kurkle/color" "^0.3.0" + chokidar@^3.5.2, chokidar@^3.5.3: version "3.5.3" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd" @@ -8385,6 +8397,11 @@ raw-body@2.5.2: iconv-lite "0.4.24" unpipe "1.0.0" +react-chartjs-2@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/react-chartjs-2/-/react-chartjs-2-5.2.0.tgz#43c1e3549071c00a1a083ecbd26c1ad34d385f5d" + integrity sha512-98iN5aguJyVSxp5U3CblRLH67J8gkfyGNbiK3c+l1QI/G4irHMPQw44aEPmjVag+YKTyQ260NcF82GTQ3bdscA== + react-colorful@^5.1.2: version "5.6.1" resolved "https://registry.yarnpkg.com/react-colorful/-/react-colorful-5.6.1.tgz#7dc2aed2d7c72fac89694e834d179e32f3da563b" From f84869925a12a8f2bd9481bfa8ba7616167fd51f Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 2 May 2023 15:43:09 +0200 Subject: [PATCH 15/45] adds description to asIds output for more info --- .../java/com/bakdata/conquery/apiv1/MetaDataPatch.java | 8 +------- .../bakdata/conquery/apiv1/query/TableExportQuery.java | 2 +- .../models/datasets/concepts/tree/ConceptTreeChild.java | 3 ++- .../models/datasets/concepts/tree/ConceptTreeNode.java | 2 ++ 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java b/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java index a78b011c8b..b96c367d9c 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java @@ -47,13 +47,7 @@ public class MetaDataPatch implements Taggable, Labelable, ShareInformation { * @param Type of the instance that is patched */ public , INST extends Taggable & Shareable & Labelable & Identifiable & Owned & Authorized> void applyTo(INST instance, MetaStorage storage, Subject subject) { - buildChain( - QueryUtils.getNoOpEntryPoint(), - storage, - subject, - instance - ) - .accept(this); + buildChain(QueryUtils.getNoOpEntryPoint(), storage, subject, instance).accept(this); } protected , INST extends Taggable & Shareable & Labelable & Identifiable & Owned & Authorized> Consumer buildChain(Consumer patchConsumerChain, MetaStorage storage, Subject subject, INST instance) { diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index 6fc57971a6..37d76badf5 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -311,7 +311,7 @@ public static String printValue(Concept concept, Object rawValue, PrintSettings return node.getId().toStringWithoutDataset(); } - return node.getName(); + return node.getName() + " - " + node.getDescription(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeChild.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeChild.java index 7e941a607d..a7ea7b1879 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeChild.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeChild.java @@ -22,7 +22,8 @@ public class ConceptTreeChild extends ConceptElement impleme @JsonIgnore private transient int[] prefix; - @JsonManagedReference //@Valid + + @JsonManagedReference @Getter @Setter private List children = Collections.emptyList(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeNode.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeNode.java index 8557d14b3a..8a6e268d46 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeNode.java @@ -26,4 +26,6 @@ public interface ConceptTreeNode Date: Tue, 2 May 2023 17:35:49 +0200 Subject: [PATCH 16/45] adds check for null descriptions --- .../com/bakdata/conquery/apiv1/query/TableExportQuery.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index 37d76badf5..8ab94340d9 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -311,7 +311,12 @@ public static String printValue(Concept concept, Object rawValue, PrintSettings return node.getId().toStringWithoutDataset(); } + if (node.getDescription() == null) { + return node.getName(); + } + return node.getName() + " - " + node.getDescription(); + } @Override From 74e5be4784338bf0b41e729b991923ab0ff667d5 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 2 May 2023 17:45:19 +0200 Subject: [PATCH 17/45] Use react-chartjs-2 to display a small chart --- frontend/src/js/entity-history/EntityCard.tsx | 151 +++------------- .../js/entity-history/TimeStratifiedChart.tsx | 169 ++++++++++++++++++ 2 files changed, 191 insertions(+), 129 deletions(-) create mode 100644 frontend/src/js/entity-history/TimeStratifiedChart.tsx diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index 23ef7b7f8b..0d4ca9d419 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -1,22 +1,14 @@ -import { useTheme } from "@emotion/react"; import styled from "@emotion/styled"; -import { - Chart as ChartJS, - CategoryScale, - LinearScale, - BarElement, - Tooltip, - ChartOptions, -} from "chart.js"; -import { Fragment, useMemo } from "react"; -import { Bar } from "react-chartjs-2"; +import { Fragment } from "react"; +import { NumericFormat } from "react-number-format"; import { useSelector } from "react-redux"; -import { EntityInfo, TimeStratifiedInfo } from "../api/types"; +import { CurrencyConfigT, EntityInfo, TimeStratifiedInfo } from "../api/types"; import { StateT } from "../app/reducers"; import { exists } from "../common/helpers/exists"; import EntityInfos from "./EntityInfos"; +import { TimeStratifiedChart } from "./TimeStratifiedChart"; import { getColumnType } from "./timeline/util"; const Container = styled("div")` @@ -32,19 +24,10 @@ const Container = styled("div")` const Centered = styled("div")` display: flex; align-items: flex-start; -`; - -const Col = styled("div")` - display: flex; flex-direction: column; gap: 10px; `; -const ChartContainer = styled("div")` - height: 200px; - width: 100%; -`; - const Grid = styled("div")` display: grid; gap: 0 20px; @@ -61,35 +44,14 @@ const Value = styled("div")` justify-self: end; `; -ChartJS.register( - CategoryScale, - LinearScale, - BarElement, - // Title, - Tooltip, - // Legend, -); - -function hexToRgbA(hex: string) { - let c: any; - if (/^#([A-Fa-f0-9]{3}){1,2}$/.test(hex)) { - c = hex.substring(1).split(""); - if (c.length === 3) { - c = [c[0], c[0], c[1], c[1], c[2], c[2]]; - } - c = "0x" + c.join(""); - return [(c >> 16) & 255, (c >> 8) & 255, c & 255].join(","); - } - throw new Error("Bad Hex"); -} - const Table = ({ timeStratifiedInfos, - currencyUnit, }: { timeStratifiedInfos: TimeStratifiedInfo[]; - currencyUnit: string; }) => { + const currencyConfig = useSelector( + (state) => state.startup.config.currency, + ); return ( <> {timeStratifiedInfos.map((timeStratifiedInfo) => { @@ -117,8 +79,17 @@ const Table = ({ - {valueFormatted} - {columnType === "MONEY" ? " " + currencyUnit : ""} + {columnType === "MONEY" && typeof value === "number" ? ( + + ) : ( + valueFormatted + )} ); @@ -130,87 +101,6 @@ const Table = ({ ); }; -const TimeStratifiedInfos = ({ - timeStratifiedInfos, -}: { - timeStratifiedInfos: TimeStratifiedInfo[]; -}) => { - const currencyUnit = useSelector( - (state) => state.startup.config.currency.unit, - ); - - const theme = useTheme(); - const datasets = useMemo(() => { - const sortedYears = [...timeStratifiedInfos[0].years].sort( - (a, b) => b.year - a.year, - ); - - return sortedYears.map((year, i) => { - return { - label: year.year.toString(), - data: Object.values(year.values), - backgroundColor: `rgba(${hexToRgbA(theme.col.blueGrayDark)}, ${1 / i})`, - }; - }); - }, [theme, timeStratifiedInfos]); - - const entries = Object.entries(timeStratifiedInfos[0].totals); - const labels = entries.map(([key]) => key); - // const values = entries.map(([, value]) => value); - - const data = { - labels, - datasets, - }; - - const options: ChartOptions<"bar"> = useMemo(() => { - return { - // indexAxis: "y" as const, - // legend: { - // position: "right" as const, - // }, - plugins: { - title: { - display: true, - text: timeStratifiedInfos[0].label, - }, - subtitle: { - text: "ololol", - }, - }, - responsive: true, - interaction: { - mode: "index" as const, - intersect: false, - }, - datasets: {}, - layout: { - padding: 0, - }, - // scales: { - // x: { - // stacked: true, - // }, - // y: { - // stacked: true, - // }, - // }, - }; - }, [timeStratifiedInfos]); - - return ( - - - - - {/*
*/} - - ); -}; - export const EntityCard = ({ className, infos, @@ -224,8 +114,11 @@ export const EntityCard = ({ +
- + ); }; diff --git a/frontend/src/js/entity-history/TimeStratifiedChart.tsx b/frontend/src/js/entity-history/TimeStratifiedChart.tsx new file mode 100644 index 0000000000..8bfdb576d9 --- /dev/null +++ b/frontend/src/js/entity-history/TimeStratifiedChart.tsx @@ -0,0 +1,169 @@ +import { useTheme } from "@emotion/react"; +import styled from "@emotion/styled"; +import { + Chart as ChartJS, + CategoryScale, + LinearScale, + BarElement, + Tooltip, + ChartOptions, +} from "chart.js"; +import { useCallback, useMemo } from "react"; +import { Bar } from "react-chartjs-2"; +import { useSelector } from "react-redux"; + +import { CurrencyConfigT, TimeStratifiedInfo } from "../api/types"; +import { StateT } from "../app/reducers"; +import { exists } from "../common/helpers/exists"; + +ChartJS.register(CategoryScale, LinearScale, BarElement, Tooltip); + +const ChartContainer = styled("div")` + height: 185px; + width: 100%; + display: flex; + justify-content: flex-end; +`; + +function hexToRgbA(hex: string) { + let c: any; + if (/^#([A-Fa-f0-9]{3}){1,2}$/.test(hex)) { + c = hex.substring(1).split(""); + if (c.length === 3) { + c = [c[0], c[0], c[1], c[1], c[2], c[2]]; + } + c = "0x" + c.join(""); + return [(c >> 16) & 255, (c >> 8) & 255, c & 255].join(","); + } + throw new Error("Bad Hex"); +} + +function interpolateDecreasingOpacity(index: number) { + return Math.min(1, 1 / (index + 0.3)); +} + +const useFormatCurrency = () => { + const currencyConfig = useSelector( + (state) => state.startup.config.currency, + ); + + const formatCurrency = useCallback( + (value: number) => { + return value.toLocaleString("de-DE", { + style: "currency", + currency: "EUR", + minimumFractionDigits: currencyConfig.decimalScale, + maximumFractionDigits: currencyConfig.decimalScale, + }); + }, + [currencyConfig], + ); + + return { + formatCurrency, + }; +}; + +export const TimeStratifiedChart = ({ + timeStratifiedInfos, +}: { + timeStratifiedInfos: TimeStratifiedInfo[]; +}) => { + const theme = useTheme(); + const infosToVisualize = timeStratifiedInfos[0]; + const labels = infosToVisualize.columns.map((col) => col.label); + + const datasets = useMemo(() => { + const sortedYears = [...infosToVisualize.years].sort( + (a, b) => b.year - a.year, + ); + + return sortedYears.map((year, i) => { + return { + label: year.year.toString(), + data: labels.map((label) => year.values[label]), + backgroundColor: `rgba(${hexToRgbA( + theme.col.blueGrayDark, + )}, ${interpolateDecreasingOpacity(i)})`, + }; + }); + }, [theme, infosToVisualize, labels]); + + const data = { + labels, + datasets, + }; + + const { formatCurrency } = useFormatCurrency(); + + const options: ChartOptions<"bar"> = useMemo(() => { + return { + plugins: { + title: { + display: true, + text: infosToVisualize.label, + }, + tooltip: { + usePointStyle: true, + backgroundColor: "rgba(255, 255, 255, 0.9)", + titleColor: "rgba(0, 0, 0, 1)", + bodyColor: "rgba(0, 0, 0, 1)", + borderColor: "rgba(0, 0, 0, 0.2)", + borderWidth: 0.5, + padding: 10, + callbacks: { + label: (context) => { + const label = context.dataset.label || context.label || ""; + const value = exists(context.parsed.y) + ? formatCurrency(context.parsed.y) + : 0; + return `${label}: ${value}`; + }, + }, + caretSize: 0, + caretPadding: 0, + }, + legend: { + labels: { + usePointStyle: true, + pointStyle: "circle", + pointBorderWidth: 1, + pointBorderColor: "rgba(0, 0, 0, 0.2)", + }, + }, + }, + responsive: true, + interaction: { + mode: "index" as const, + intersect: false, + }, + layout: { + padding: 0, + }, + maintainAspectRatio: false, + scales: { + x: { + ticks: { + callback: (idx: any) => { + return labels[idx].length > 12 + ? labels[idx].substring(0, 9) + "..." + : labels[idx]; + }, + }, + }, + y: { + ticks: { + callback: (value) => + typeof value === "number" ? formatCurrency(value) : value, + }, + }, + }, + }; + }, [infosToVisualize, labels, formatCurrency]); + + return ( + + + + ); +}; From 67ac86700c6c48f65ad4c8485b2b0c47fee81a97 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 3 May 2023 12:50:32 +0200 Subject: [PATCH 18/45] uses ExecutionId in ExecutionManager cache, because the underlying implementation relies on hashing which breaks results on patch --- .../conquery/apiv1/QueryProcessor.java | 20 +-------- .../io/result/csv/ResultCsvProcessor.java | 7 +-- .../conquery/io/storage/MetaStorage.java | 12 ++--- .../models/query/ExecutionManager.java | 44 ++++++++++--------- .../integration/common/LoadingUtil.java | 4 +- 5 files changed, 33 insertions(+), 54 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 8021c91643..4a9e91d0f9 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -184,7 +184,7 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId exe if (!user.isOwner(execution)) { final ManagedExecution newExecution = - executionManager.createExecution(namespace, execution.getSubmitted(), user, execution.getDataset(), false); + executionManager.createExecution(execution.getSubmitted(), user, execution.getDataset(), false); newExecution.setLabel(execution.getLabel()); newExecution.setTags(execution.getTags().clone()); storage.updateExecution(newExecution); @@ -318,22 +318,6 @@ public void patchQuery(Subject subject, ManagedExecution execution, MetaDataPatc patch.applyTo(execution, storage, subject); storage.updateExecution(execution); - - // TODO remove this, since we don't translate anymore - // Patch this query in other datasets - final List remainingDatasets = datasetRegistry.getAllDatasets(); - remainingDatasets.remove(execution.getDataset()); - - for (Dataset dataset : remainingDatasets) { - final ManagedExecutionId id = new ManagedExecutionId(dataset.getId(), execution.getQueryId()); - final ManagedExecution otherExecution = storage.getExecution(id); - if (otherExecution == null) { - continue; - } - log.trace("Patching {} ({}) with patch: {}", execution.getClass().getSimpleName(), id, patch); - patch.applyTo(otherExecution, storage, subject); - storage.updateExecution(execution); - } } public void reexecute(Subject subject, ManagedExecution query) { @@ -399,7 +383,7 @@ public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, Ext execution = ((ManagedQuery) namespace .getExecutionManager() - .createExecution(namespace, query, subject.getUser(), dataset, false)); + .createExecution(query, subject.getUser(), dataset, false)); execution.setLastResultCount((long) statistic.getResolved().size()); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index 490337eddd..543cf3a195 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -75,12 +75,7 @@ public Response createResult(Su } }; - return makeResponseWithFileName( - Response.ok(out), - String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_CSV), - new MediaType("text", "csv", charset.toString()), - ResultUtil.ContentDispositionOption.ATTACHMENT - ); + return makeResponseWithFileName(Response.ok(out), String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_CSV), new MediaType("text", "csv", charset.toString()), ResultUtil.ContentDispositionOption.ATTACHMENT); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index b07a579087..12e283dfaa 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -30,20 +30,17 @@ @RequiredArgsConstructor public class MetaStorage extends ConqueryStorage implements Injectable { + @Getter + protected final CentralRegistry centralRegistry = new CentralRegistry(); + @Getter + protected final DatasetRegistry datasetRegistry; private final StoreFactory storageFactory; - private IdentifiableStore executions; - private IdentifiableStore formConfigs; private IdentifiableStore authUser; private IdentifiableStore authRole; private IdentifiableStore authGroup; - @Getter - protected final CentralRegistry centralRegistry = new CentralRegistry(); - @Getter - protected final DatasetRegistry datasetRegistry; - public void openStores(ObjectMapper mapper) { authUser = storageFactory.createUserStore(centralRegistry, "meta", this, mapper); authRole = storageFactory.createRoleStore(centralRegistry, "meta", this, mapper); @@ -66,7 +63,6 @@ public void openStores(ObjectMapper mapper) { authUser, authRole, authGroup, - executions, formConfigs ); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java index d075834b09..ea2e844612 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java @@ -17,6 +17,7 @@ import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.InternalExecution; import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.ShardResult; import com.bakdata.conquery.models.worker.Namespace; @@ -33,36 +34,41 @@ public class ExecutionManager { private final MetaStorage storage; - private final Cache>> executionResults = CacheBuilder.newBuilder() - .softValues() - .removalListener(this::executionRemoved) - .build(); + private final Cache>> executionResults = + CacheBuilder.newBuilder() + .softValues() + .removalListener(this::executionRemoved) + .build(); + /** * Manage state of evicted Queries, setting them to NEW. */ - private void executionRemoved(RemovalNotification> removalNotification) { + private void executionRemoved(RemovalNotification> removalNotification) { // If removal was done manually we assume it was also handled properly if (!removalNotification.wasEvicted()) { return; } - final ManagedExecution execution = removalNotification.getKey(); + final ManagedExecutionId executionId = removalNotification.getKey(); - log.warn("Evicted Results for Query[{}] (Reason: {})", execution.getId(), removalNotification.getCause()); + log.warn("Evicted Results for Query[{}] (Reason: {})", executionId, removalNotification.getCause()); - execution.reset(); + storage.getExecution(executionId).reset(); } - public ManagedExecution runQuery(Namespace namespace, QueryDescription query, User user, Dataset submittedDataset, ConqueryConfig config, boolean system) { - final ManagedExecution execution = createExecution(namespace, query, user, submittedDataset, system); + final ManagedExecution execution = createExecution(query, user, submittedDataset, system); execute(namespace, execution, config); return execution; } + public ManagedExecution createExecution(QueryDescription query, User user, Dataset submittedDataset, boolean system) { + return createQuery(query, UUID.randomUUID(), user, submittedDataset, system); + } + public void execute(Namespace namespace, ManagedExecution execution, ConqueryConfig config) { // Initialize the query / create subqueries try { @@ -90,12 +96,8 @@ public void execute(Namespace namespace, ManagedExecution execution, ConqueryCon } } - public ManagedExecution createExecution(Namespace namespace, QueryDescription query, User user, Dataset submittedDataset, boolean system) { - return createQuery(namespace, query, UUID.randomUUID(), user, submittedDataset, system); - } - - public ManagedExecution createQuery(Namespace namespace, QueryDescription query, UUID queryId, User user, Dataset submittedDataset, boolean system) { + public ManagedExecution createQuery(QueryDescription query, UUID queryId, User user, Dataset submittedDataset, boolean system) { // Transform the submitted query into an initialized execution ManagedExecution managed = query.toManagedExecution(user, submittedDataset, storage); managed.setSystem(system); @@ -107,7 +109,6 @@ public ManagedExecution createQuery(Namespace namespace, QueryDescription query, return managed; } - /** * Receive part of query result and store into query. * @@ -134,14 +135,13 @@ public } } - /** * Register another result for the execution. */ @SneakyThrows(ExecutionException.class) // can only occur if ArrayList::new fails which is unlikely and would have other problems also public void addQueryResult(ManagedExecution execution, List queryResults) { // We don't collect all results together into a fat list as that would cause lots of huge re-allocations for little gain. - executionResults.get(execution, ArrayList::new) + executionResults.get(execution.getId(), ArrayList::new) .add(queryResults); } @@ -149,17 +149,21 @@ public void addQueryResult(ManagedExecution execution, List queryR * Discard the query's results. */ public void clearQueryResults(ManagedExecution execution) { - executionResults.invalidate(execution); + executionResults.invalidate(execution.getId()); } /** * Stream the results of the query, if available. */ public Stream streamQueryResults(ManagedExecution execution) { - final List> resultParts = executionResults.getIfPresent(execution); + final List> resultParts = executionResults.getIfPresent(execution.getId()); return resultParts == null ? Stream.empty() : resultParts.stream().flatMap(List::stream); } + + + + } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java index 20833bd437..c8ada30207 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java @@ -74,7 +74,7 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData ConceptQuery query = new ConceptQuery(new CQExternal(Arrays.asList("ID", "DATE_SET"), data, false)); ManagedExecution managed = support.getNamespace().getExecutionManager() - .createQuery(support.getNamespace(), query, queryId, user, support.getNamespace().getDataset(), false); + .createQuery(query, queryId, user, support.getNamespace().getDataset(), false); user.addPermission(managed.createPermission(AbilitySets.QUERY_CREATOR)); @@ -91,7 +91,7 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData ManagedExecution managed = support.getNamespace() .getExecutionManager() - .createQuery(support.getNamespace(), query, queryId, user, support.getNamespace().getDataset(), false); + .createQuery(query, queryId, user, support.getNamespace().getDataset(), false); user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managed.getId())); From 2f9ade901a36c2c36ed868e82084680593c4801d Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 3 May 2023 12:56:37 +0200 Subject: [PATCH 19/45] fixes order of fields, to avoid having to reorder all callers --- .../main/java/com/bakdata/conquery/io/storage/MetaStorage.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index 12e283dfaa..ee294b92d8 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -32,9 +32,10 @@ public class MetaStorage extends ConqueryStorage implements Injectable { @Getter protected final CentralRegistry centralRegistry = new CentralRegistry(); + private final StoreFactory storageFactory; + @Getter protected final DatasetRegistry datasetRegistry; - private final StoreFactory storageFactory; private IdentifiableStore executions; private IdentifiableStore formConfigs; private IdentifiableStore authUser; From 410e3c6f7fee4610bc48ee57c363695f3e87f6d8 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 8 May 2023 09:03:40 +0200 Subject: [PATCH 20/45] change hover navitgable Timeout from 0.6s to 1s --- frontend/src/js/small-tab-navigation/HoverNavigatable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx index 20aa1946e3..1f70870c3b 100644 --- a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx +++ b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx @@ -33,7 +33,7 @@ const Root = styled("div")<{ `; // estimated to feel responsive, but not too quick -const TIME_UNTIL_NAVIGATE = 600; +const TIME_UNTIL_NAVIGATE = 1000; export const HoverNavigatable = ({ triggerNavigate, From fcfe9135bb60534fc1e5a2e6abbc84f267c00496 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 8 May 2023 11:26:59 +0200 Subject: [PATCH 21/45] Disable table use for a moment --- frontend/src/js/entity-history/EntityCard.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index 0d4ca9d419..d91f481be7 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -44,6 +44,8 @@ const Value = styled("div")` justify-self: end; `; +// @ts-ignore EVALUATE IF WE WANT TO SHOW THIS TABLE WITH FUTURE DATA +// eslint-disable-next-line @typescript-eslint/no-unused-vars const Table = ({ timeStratifiedInfos, }: { @@ -114,7 +116,8 @@ export const EntityCard = ({ -
+ {/* TODO: EVALUATE IF WE WANT TO SHOW THIS TABLE WITH FUTURE DATA +
*/} Date: Tue, 9 May 2023 15:29:34 +0200 Subject: [PATCH 22/45] cleanup --- .../concepts/FrontEndConceptBuilder.java | 128 +++++++++--------- 1 file changed, 65 insertions(+), 63 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java index edbf99b1c3..d453f7e981 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java @@ -46,10 +46,10 @@ public class FrontEndConceptBuilder { public static FrontendRoot createRoot(NamespaceStorage storage, Subject subject) { - FrontendRoot root = new FrontendRoot(); - Map, FrontendNode> roots = root.getConcepts(); + final FrontendRoot root = new FrontendRoot(); + final Map, FrontendNode> roots = root.getConcepts(); - List> allConcepts = new ArrayList<>(storage.getAllConcepts()); + final List> allConcepts = new ArrayList<>(storage.getAllConcepts()); // Remove any hidden concepts allConcepts.removeIf(Concept::isHidden); @@ -58,22 +58,24 @@ public static FrontendRoot createRoot(NamespaceStorage storage, Subject subject) } // Submit all permissions to Shiro - boolean[] isPermitted = subject.isPermitted(allConcepts, Ability.READ); + final boolean[] isPermitted = subject.isPermitted(allConcepts, Ability.READ); - for (int i = 0; i c, StructureNode[] structureNodes) { - MatchingStats matchingStats = c.getMatchingStats(); + final MatchingStats matchingStats = c.getMatchingStats(); - StructureNodeId structureParent = Arrays + final StructureNodeId structureParent = Arrays .stream(structureNodes) .filter(sn -> sn.getContainedRoots().contains(c.getId())) .findAny() .map(StructureNode::getId) .orElse(null); - FrontendNode n = FrontendNode.builder() - .active(true) - .description(c.getDescription()) - .label(c.getLabel()) - .additionalInfos(c.getAdditionalInfos()) - .matchingEntries(matchingStats != null ? matchingStats.countEvents() : 0) - .matchingEntities(matchingStats != null ? matchingStats.countEntities() : 0) - .dateRange(matchingStats != null && matchingStats.spanEvents() != null ? matchingStats.spanEvents().toSimpleRange() : null) - .detailsAvailable(Boolean.TRUE) - .codeListResolvable(c.countElements() > 1) - .parent(structureParent) - .selects(c + final FrontendNode n = FrontendNode.builder() + .active(true) + .description(c.getDescription()) + .label(c.getLabel()) + .additionalInfos(c.getAdditionalInfos()) + .matchingEntries(matchingStats != null ? matchingStats.countEvents() : 0) + .matchingEntities(matchingStats != null ? matchingStats.countEntities() : 0) + .dateRange(matchingStats != null && matchingStats.spanEvents() != null ? matchingStats.spanEvents().toSimpleRange() : null) + .detailsAvailable(Boolean.TRUE) + .codeListResolvable(c.countElements() > 1) + .parent(structureParent) + .selects(c .getSelects() .stream() .map(FrontEndConceptBuilder::createSelect) .collect(Collectors.toList()) ) - .tables(c + .tables(c .getConnectors() .stream() .map(FrontEndConceptBuilder::createTable) .collect(Collectors.toList()) ) - .build(); + .build(); if (c instanceof ConceptTreeNode tree && tree.getChildren() != null) { n.setChildren( @@ -140,7 +142,7 @@ private static FrontendNode createCTRoot(Concept c, StructureNode[] structure @Nullable private static FrontendNode createStructureNode(StructureNode cn, Map, FrontendNode> roots) { - List unstructured = new ArrayList<>(); + final List unstructured = new ArrayList<>(); for (ConceptId id : cn.getContainedRoots()) { if (!roots.containsKey(id)) { log.trace("Concept from structure node can not be found: {}", id); @@ -170,31 +172,19 @@ private static FrontendNode createStructureNode(StructureNode cn, Map, Fro .build(); } - private static FrontendNode createCTNode(ConceptElement ce) { - MatchingStats matchingStats = ce.getMatchingStats(); - FrontendNode n = FrontendNode.builder() - .active(null) - .description(ce.getDescription()) - .label(ce.getLabel()) - .additionalInfos(ce.getAdditionalInfos()) - .matchingEntries(matchingStats != null ? matchingStats.countEvents() : 0) - .matchingEntities(matchingStats != null ? matchingStats.countEntities() : 0) - .dateRange(matchingStats != null && matchingStats.spanEvents() != null ? matchingStats.spanEvents().toSimpleRange() : null) - .build(); - - if (ce instanceof ConceptTreeNode tree) { - if (tree.getChildren() != null) { - n.setChildren(tree.getChildren().stream().map(IdentifiableImpl::getId).toArray(ConceptTreeChildId[]::new)); - } - if (tree.getParent() != null) { - n.setParent(tree.getParent().getId()); - } - } - return n; + public static FrontendSelect createSelect(Select select) { + return FrontendSelect + .builder() + .id(select.getId()) + .label(select.getLabel()) + .description(select.getDescription()) + .resultType(select.getResultType()) + .isDefault(select.isDefault()) + .build(); } public static FrontendTable createTable(Connector con) { - FrontendTable result = + final FrontendTable result = FrontendTable.builder() .id(con.getTable().getId()) .connectorId(con.getId()) @@ -221,7 +211,7 @@ public static FrontendTable createTable(Connector con) { ) .build(); - if(con.getValidityDates().size() > 1) { + if (con.getValidityDates().size() > 1) { result.setDateColumn( new FrontendValidityDate( con.getValidityDatesDescription(), @@ -234,7 +224,7 @@ public static FrontendTable createTable(Connector con) { ) ); - if(!result.getDateColumn().getOptions().isEmpty()) { + if (!result.getDateColumn().getOptions().isEmpty()) { result.getDateColumn().setDefaultValue(result.getDateColumn().getOptions().get(0).getValue()); } } @@ -251,19 +241,31 @@ public static FrontendFilterConfiguration.Top createFilter(Filter filter) { } } - public static FrontendSelect createSelect(Select select) { - return FrontendSelect - .builder() - .id(select.getId()) - .label(select.getLabel()) - .description(select.getDescription()) - .resultType(select.getResultType()) - .isDefault(select.isDefault()) - .build(); + private static FrontendNode createCTNode(ConceptElement ce) { + final MatchingStats matchingStats = ce.getMatchingStats(); + final FrontendNode n = FrontendNode.builder() + .active(null) + .description(ce.getDescription()) + .label(ce.getLabel()) + .additionalInfos(ce.getAdditionalInfos()) + .matchingEntries(matchingStats != null ? matchingStats.countEvents() : 0) + .matchingEntities(matchingStats != null ? matchingStats.countEntities() : 0) + .dateRange(matchingStats != null && matchingStats.spanEvents() != null ? matchingStats.spanEvents().toSimpleRange() : null) + .build(); + + if (ce instanceof ConceptTreeNode tree) { + if (tree.getChildren() != null) { + n.setChildren(tree.getChildren().stream().map(IdentifiableImpl::getId).toArray(ConceptTreeChildId[]::new)); + } + if (tree.getParent() != null) { + n.setParent(tree.getParent().getId()); + } + } + return n; } public static FrontendList createTreeMap(Concept concept) { - FrontendList map = new FrontendList(); + final FrontendList map = new FrontendList(); fillTreeMap(concept, map); return map; } From cf5863fd9c8a4847f9f44b063e0701e15655d1e8 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 9 May 2023 17:05:02 +0200 Subject: [PATCH 23/45] adds `excludeFromTimeAggregation` and `defaultExcludeFromTimeAggregation` for Concepts --- .../conquery/apiv1/frontend/FrontendNode.java | 1 + .../models/datasets/concepts/Concept.java | 13 +- .../concepts/FrontEndConceptBuilder.java | 169 +++++++----------- 3 files changed, 73 insertions(+), 110 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendNode.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendNode.java index 318e0bc9c0..6eeace5595 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendNode.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendNode.java @@ -32,4 +32,5 @@ public class FrontendNode { private boolean codeListResolvable; private List selects; private long matchingEntities; + private boolean excludeFromTimeAggregation; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java index e18fcd195c..d4c51d7c3e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java @@ -40,7 +40,7 @@ */ @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") @CPSBase -@ToString(of = {"connectors"}) +@ToString(of = "connectors") @Getter @Setter @EqualsAndHashCode(callSuper = true) @@ -49,7 +49,9 @@ public abstract class Concept extends ConceptElemen /** * Display Concept for users. */ - private boolean hidden = false; + private boolean hidden; + + private boolean defaultExcludeFromTimeAggregation = false; @JsonManagedReference @Valid @@ -59,10 +61,7 @@ public abstract class Concept extends ConceptElemen private Dataset dataset; public List
-
- 
${job.progressReporter.estimate}
 +
From c6b76b48ef63f4acdbe7eae7cfceecd3e021b24a Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Thu, 11 May 2023 10:08:55 +0200 Subject: [PATCH 27/45] added year and month selection to Datepicker --- .../ui-components/InputDate/CustomHeader.tsx | 150 ++++++++++++++++++ .../InputDate/CustomHeaderComponents.tsx | 41 +++++ .../{ => InputDate}/InputDate.tsx | 7 +- .../src/js/ui-components/InputDateRange.tsx | 2 +- 4 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 frontend/src/js/ui-components/InputDate/CustomHeader.tsx create mode 100644 frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx rename frontend/src/js/ui-components/{ => InputDate}/InputDate.tsx (92%) diff --git a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx new file mode 100644 index 0000000000..5f5c1a89a2 --- /dev/null +++ b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx @@ -0,0 +1,150 @@ +import { + faChevronLeft, + faChevronRight, +} from "@fortawesome/free-solid-svg-icons"; +import { useState } from "react"; +import { ReactDatePickerCustomHeaderProps } from "react-datepicker"; + +import { SelectOptionT } from "../../api/types"; +import IconButton from "../../button/IconButton"; +import { Menu } from "../InputSelect/InputSelectComponents"; + +import { + MonthYearLabel, + OptionButton, + OptionList, + Root, + SelectMenuContainer, +} from "./CustomHeaderComponents"; + +const yearOptions: SelectOptionT[] = [...Array(10).keys()] + .map((n) => new Date().getFullYear() - n) + .map((year) => ({ + label: String(year), + value: year, + })) + .reverse(); + +const months = [ + "Januar", + "Februar", + "März", + "April", + "Mai", + "Juni", + "Juli", + "August", + "September", + "Oktober", + "November", + "Dezember", +]; +const monthOptions: SelectOptionT[] = months.map((month, i) => ({ + label: month, + value: i, +})); + +const SelectMenu = ({ + date, + options, + onSelect, +}: Pick & { + options: SelectOptionT[]; + onSelect: (n: number) => void; +}) => ( + + + + {options.map((option) => ( + onSelect(option.value as number)} + > + {option.label} + + ))} + + + +); + +const YearMonthSelect = ({ + date, + changeMonth, + changeYear, +}: Pick< + ReactDatePickerCustomHeaderProps, + "date" | "changeYear" | "changeMonth" +>) => { + const [yearSelectOpen, setYearSelectOpen] = useState(false); + const [monthSelectOpen, setMonthSelectOpen] = useState(false); + const handleClick = () => { + if (yearSelectOpen || monthSelectOpen) { + setYearSelectOpen(false); + setMonthSelectOpen(false); + } else { + setYearSelectOpen(true); + } + }; + + return ( + <> + + {months[date.getMonth()]} {date.getFullYear()} + + {yearSelectOpen && ( + { + changeYear(year); + setYearSelectOpen(false); + setMonthSelectOpen(true); + }} + /> + )} + {monthSelectOpen && ( + { + changeMonth(month); + setMonthSelectOpen(false); + }} + /> + )} + + ); +}; + +export const CustomHeader = ({ + date, + changeYear, + changeMonth, + decreaseMonth, + increaseMonth, + prevMonthButtonDisabled, + nextMonthButtonDisabled, +}: ReactDatePickerCustomHeaderProps) => { + return ( + + + + + + ); +}; diff --git a/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx b/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx new file mode 100644 index 0000000000..f669e40aae --- /dev/null +++ b/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx @@ -0,0 +1,41 @@ +import styled from "@emotion/styled"; + +import BasicButton from "../../button/BasicButton"; +import { List } from "../InputSelect/InputSelectComponents"; + +export const Root = styled("div")` + display: flex; + justify-content: space-between; + align-items: center; +`; + +export const SelectMenuContainer = styled("div")` + position: absolute; + top: 40px; + left: 0; + right: 0; +`; + +export const OptionList = styled(List)` + display: grid; + grid-template-columns: auto auto; + gap: 5px; +`; + +export const OptionButton = styled(BasicButton)` + font-size: 14px; + background: ${({ theme, active }) => + active ? theme.col.blueGrayDark : "inherit"}; + color: ${({ active }) => (active ? "white" : "inherit")}; + border-radius: ${({ theme }) => theme.borderRadius}; + border: ${({ theme }) => "1px solid " + theme.col.gray}; + &:hover { + background: ${({ theme }) => theme.col.blueGrayDark}; + color: white; + } +`; + +export const MonthYearLabel = styled("div")` + font-weight: bold; + cursor: pointer; +`; diff --git a/frontend/src/js/ui-components/InputDate.tsx b/frontend/src/js/ui-components/InputDate/InputDate.tsx similarity index 92% rename from frontend/src/js/ui-components/InputDate.tsx rename to frontend/src/js/ui-components/InputDate/InputDate.tsx index 3bd58ce481..99cc141a3d 100644 --- a/frontend/src/js/ui-components/InputDate.tsx +++ b/frontend/src/js/ui-components/InputDate/InputDate.tsx @@ -3,9 +3,10 @@ import { createElement, forwardRef, useRef, useState } from "react"; import ReactDatePicker from "react-datepicker"; import "react-datepicker/dist/react-datepicker.css"; -import { formatDate, parseDate } from "../common/helpers/dateHelper"; +import { formatDate, parseDate } from "../../common/helpers/dateHelper"; +import BaseInput, { Props as BaseInputProps } from "../BaseInput"; -import BaseInput, { Props as BaseInputProps } from "./BaseInput"; +import { CustomHeader } from "./CustomHeader"; const Root = styled("div")` position: relative; @@ -121,8 +122,10 @@ const InputDate = forwardRef( datePickerRef.current?.setOpen(false); }} onClickOutside={() => datePickerRef.current?.setOpen(false)} + renderCustomHeader={CustomHeader} customInput={createElement(HiddenInput)} calendarContainer={StyledCalendar} + showFullMonthYearPicker={true} /> ); diff --git a/frontend/src/js/ui-components/InputDateRange.tsx b/frontend/src/js/ui-components/InputDateRange.tsx index e14e0458b8..f8e0e46662 100644 --- a/frontend/src/js/ui-components/InputDateRange.tsx +++ b/frontend/src/js/ui-components/InputDateRange.tsx @@ -14,7 +14,7 @@ import { import { exists } from "../common/helpers/exists"; import InfoTooltip from "../tooltip/InfoTooltip"; -import InputDate from "./InputDate"; +import InputDate from "./InputDate/InputDate"; import Label from "./Label"; import Labeled from "./Labeled"; import Optional from "./Optional"; From 063cb45ed70ca1215cfa59dd098de6fc3993ae3e Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Thu, 11 May 2023 10:24:42 +0200 Subject: [PATCH 28/45] removed unneccessary line --- frontend/src/js/ui-components/InputDate/InputDate.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/js/ui-components/InputDate/InputDate.tsx b/frontend/src/js/ui-components/InputDate/InputDate.tsx index 99cc141a3d..a3cf0aeb52 100644 --- a/frontend/src/js/ui-components/InputDate/InputDate.tsx +++ b/frontend/src/js/ui-components/InputDate/InputDate.tsx @@ -125,7 +125,6 @@ const InputDate = forwardRef( renderCustomHeader={CustomHeader} customInput={createElement(HiddenInput)} calendarContainer={StyledCalendar} - showFullMonthYearPicker={true} /> ); From 632f3e2f5a1676a30a6017b211596d557455fcb7 Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Thu, 11 May 2023 17:01:18 +0200 Subject: [PATCH 29/45] changed absolute positioning to anchor-dimensions pattern --- .../src/js/ui-components/InputDate/CustomHeaderComponents.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx b/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx index f669e40aae..72d159246a 100644 --- a/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx +++ b/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx @@ -13,7 +13,7 @@ export const SelectMenuContainer = styled("div")` position: absolute; top: 40px; left: 0; - right: 0; + width: 100%; `; export const OptionList = styled(List)` From 9434a2596e544d70a356452692e0da29003839e9 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 15 May 2023 10:35:29 +0200 Subject: [PATCH 30/45] Documentation for .import.json (#3062) Documentation for .import.json --- openapi.yaml | 176 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 166 insertions(+), 10 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index 7f4d1e577b..d1c019e358 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -130,6 +130,160 @@ components: type: array items: $ref: "#/components/schemas/Column" + + OutputDescriptionBase: + type: object + properties: + name: + description: "Name of the `table.column` to be imported into." + type: string + required: + type: boolean + LineOutput: + description: "Emits the line number. The line-number is not guaranteed to be in file order, just an incrementing unique number: If multiple files are preprocessed, the line-number will not reset to zero." + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + operation: + type: string + enum: [ LINE ] + NullOutput: + description: "Emits a `null`/NA value. Can be used to unify among multiple input sources with differing file schemas." + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + operation: + type: string + enum: [ NULL ] + + CopyOutput: + description: "Parses and emits the values in inputColumn as the provided type." + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + inputColumn: + type: string + description: "Name of the csv column to process." + inputType: + $ref: "#/components/schemas/MajorType" + operation: + type: string + enum: [ COPY ] + CompoundDateRangeOutput: + description: "Creates a virtual daterange column as a composite of its neighboring DATE-columns (not csv columns). + This helps avoid copying multiple values of the same column, when they can be reused." + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + startColumn: + type: string + endColumn: + type: string + allowOpen: + type: boolean + operation: + type: string + enum: [ COMPOUND_DATE_RANGE ] + + DateRangeOutput: + description: "Parses `startColumn` and `endColumn` as dates, then emits them as a daterange." + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + startColumn: + type: string + endColumn: + type: string + allowOpen: + type: boolean + operation: + type: string + enum: [ DATE_RANGE ] + + EpochDateRangeOutput: + deprecated: true + description: "Parses `startColumn` and `endColumn` as integers, interpreting them as dates in the Unix-Epoch (i.e. days since 01-01-1970), then emits them as a daterange." + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + startColumn: + type: string + endColumn: + type: string + allowOpen: + type: boolean + operation: + type: string + enum: [ EPOCH_DATE_RANGE ] + + EpochOutput: + deprecated: true + description: "Parses `startColumn` integer, emitting it as dates in the Unix-Epoch (i.e. days since 01-01-1970)" + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + inputColumn: + type: string + operation: + type: string + enum: [ EPOCH ] + + OutputDescription: + description: "Operations to transform, then import, values into tables." + discriminator: + propertyName: operation + mapping: { + # FK: I'm using JSON notation here, because the key `NULL` breaks some tools. + "COMPOUND_DATE_RANGE": "#/components/schemas/CompoundDateRangeOutput", + "COPY": "#/components/schemas/CopyOutput", + "DATE_RANGE": "#/components/schemas/DateRangeOutput", + "EPOCH": "#/components/schemas/EpochOutput", + "EPOCH_DATE_RANGE": "#/components/schemas/EpochDateRangeOutput", + "LINE": "#/components/schemas/LineOutput", + "NULL": "#/components/schemas/NullOutput", + } + oneOf: + - $ref: "#/components/schemas/CompoundDateRangeOutput" + - $ref: "#/components/schemas/CopyOutput" + - $ref: "#/components/schemas/DateRangeOutput" + - $ref: "#/components/schemas/EpochDateRangeOutput" + - $ref: "#/components/schemas/EpochOutput" + - $ref: "#/components/schemas/LineOutput" + - $ref: "#/components/schemas/NullOutput" + + Import: + description: "Describes how one or multiple CSV files are to be transformed into a .cqpp file compatible with the designated table." + type: object + properties: + table: + $ref: "#/components/schemas/TableId" + name: + description: "Name of the import. When preprocessed with tags, will be overwritten to the respective tag." + type: string + label: + deprecated: true + type: string + inputs: + description: "One or more descriptions to preprocess a `.csv` or `.csv.gz` into a single cqpp." + type: array + items: + type: object + properties: + sourceFile: + description: "Filename of the to preprocess file, relative to the path supplied in `preprocess`." + type: string + filter: + description: "Groovy script which can be used to filter the csv rows. + The row is passed as `row` array, the fields can be accessed as integer variables to the index in the file. + For example accessing `value` in `id,value` can be done with `row[value]` (no string up-ticks), or `row[1]`." + type: string + primary: + $ref: "#/components/schemas/OutputDescription" + output: + type: array + items: + $ref: "#/components/schemas/OutputDescription" + FrontendDatasetId: type: object properties: @@ -1026,20 +1180,22 @@ components: type: string value: type: string - + MajorType: + type: string + enum: + - STRING + - INTEGER + - BOOLEAN + - NUMERIC + - MONEY + - DATE + - DATE_RANGE ColumnType: description: Available types for input and output of conquery. Additionally list, which is a nested type of ColumnType. oneOf: + - $ref: "#/components/schemas/MajorType" - type: string - enum: - - STRING - - INTEGER - - BOOLEAN - - NUMERIC - - MONEY - - DATE - - DATE_RANGE - - type: string + description: "Arbitrary nesting of ColumnTypes including List." pattern: "LIST\\[.*\\]" SemanticsEventDate: description: Values contain the primary date of a row. From add6dd3a8f5075221cd741e1187e5d5f880a6f36 Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Tue, 16 May 2023 15:42:00 +0200 Subject: [PATCH 31/45] add function to retrieve month names, move styled components inside CustomHeader.tsx, add constant for year selection span and reuse TransparentButton component --- frontend/src/js/common/helpers/dateHelper.ts | 14 ++++ .../ui-components/InputDate/CustomHeader.tsx | 65 +++++++++++-------- .../InputDate/CustomHeaderComponents.tsx | 41 ------------ 3 files changed, 51 insertions(+), 69 deletions(-) delete mode 100644 frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx diff --git a/frontend/src/js/common/helpers/dateHelper.ts b/frontend/src/js/common/helpers/dateHelper.ts index 789f669afc..b522810f02 100644 --- a/frontend/src/js/common/helpers/dateHelper.ts +++ b/frontend/src/js/common/helpers/dateHelper.ts @@ -10,6 +10,7 @@ import { formatDistance, } from "date-fns"; import { de, enGB } from "date-fns/locale"; +import i18next from "i18next"; import { useTranslation } from "react-i18next"; // To save the date in this format in the state @@ -213,3 +214,16 @@ export function getFirstAndLastDateOfRange(dateRangeStr: string): { return { first, last }; } + +export function getMonthName(date: Date): string { + const locale = i18next.language === "de" ? de : enGB; + return format(date, "LLLL", { locale }); +} + +export function getMonthNames(): string[] { + return [...Array(12).keys()].map((month) => { + const date = new Date(); + date.setMonth(month); + return getMonthName(date); + }); +} diff --git a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx index 5f5c1a89a2..71071a3732 100644 --- a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx +++ b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx @@ -1,3 +1,4 @@ +import styled from "@emotion/styled"; import { faChevronLeft, faChevronRight, @@ -7,17 +8,36 @@ import { ReactDatePickerCustomHeaderProps } from "react-datepicker"; import { SelectOptionT } from "../../api/types"; import IconButton from "../../button/IconButton"; -import { Menu } from "../InputSelect/InputSelectComponents"; +import { TransparentButton } from "../../button/TransparentButton"; +import { getMonthName, getMonthNames } from "../../common/helpers/dateHelper"; +import { List, Menu } from "../InputSelect/InputSelectComponents"; -import { - MonthYearLabel, - OptionButton, - OptionList, - Root, - SelectMenuContainer, -} from "./CustomHeaderComponents"; +export const Root = styled("div")` + display: flex; + justify-content: space-between; + align-items: center; +`; + +export const SelectMenuContainer = styled("div")` + position: absolute; + top: 40px; + left: 0; + width: 100%; +`; + +export const OptionList = styled(List)` + display: grid; + grid-template-columns: auto auto; + gap: 5px; +`; + +export const MonthYearLabel = styled("div")` + font-weight: bold; + cursor: pointer; +`; -const yearOptions: SelectOptionT[] = [...Array(10).keys()] +const yearSelectionSpan = 10; +const yearOptions: SelectOptionT[] = [...Array(yearSelectionSpan).keys()] .map((n) => new Date().getFullYear() - n) .map((year) => ({ label: String(year), @@ -25,21 +45,7 @@ const yearOptions: SelectOptionT[] = [...Array(10).keys()] })) .reverse(); -const months = [ - "Januar", - "Februar", - "März", - "April", - "Mai", - "Juni", - "Juli", - "August", - "September", - "Oktober", - "November", - "Dezember", -]; -const monthOptions: SelectOptionT[] = months.map((month, i) => ({ +const monthOptions: SelectOptionT[] = getMonthNames().map((month, i) => ({ label: month, value: i, })); @@ -56,14 +62,17 @@ const SelectMenu = ({ {options.map((option) => ( - onSelect(option.value as number)} > {option.label} - + ))} @@ -92,7 +101,7 @@ const YearMonthSelect = ({ return ( <> - {months[date.getMonth()]} {date.getFullYear()} + {getMonthName(date)} {date.getFullYear()} {yearSelectOpen && ( - active ? theme.col.blueGrayDark : "inherit"}; - color: ${({ active }) => (active ? "white" : "inherit")}; - border-radius: ${({ theme }) => theme.borderRadius}; - border: ${({ theme }) => "1px solid " + theme.col.gray}; - &:hover { - background: ${({ theme }) => theme.col.blueGrayDark}; - color: white; - } -`; - -export const MonthYearLabel = styled("div")` - font-weight: bold; - cursor: pointer; -`; From 79015c2e3ebdb62ac09e9247d749a98890e753c7 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 17 May 2023 11:51:45 +0200 Subject: [PATCH 32/45] set prettyPrint=false for EntityPreviewExecution output (#3072) --- .../conquery/models/query/preview/EntityPreviewExecution.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index 5963941685..6010af221a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -268,7 +268,7 @@ public FullExecutionStatus buildStatusFull(Subject subject) { setStatusFull(status, subject); status.setQuery(getValuesQuery().getQuery()); - final PrintSettings printSettings = new PrintSettings(true, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel); + final PrintSettings printSettings = new PrintSettings(false, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel); status.setInfos(transformQueryResultToInfos(getInfoCardExecution(), printSettings)); From 7b23878a3e55b5ddf931add4a0103f86410f9ca2 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 17 May 2023 13:32:28 +0200 Subject: [PATCH 33/45] Test build images in ci (#3073) * Cleanup of unsued files * adds tasks testing building of frontend and backend container --- .../workflows/test_build_backend_image.yml | 45 +++ .../workflows/test_build_frontend_image.yml | 40 +++ .restyled.yaml | 19 -- checkstyle.xml | 163 --------- heroku.yml | 4 - utilities/eclipse.cleanup.xml | 61 ---- utilities/eclipse.formatter.xml | 315 ------------------ utilities/intellij.formatter.xml | 58 ---- 8 files changed, 85 insertions(+), 620 deletions(-) create mode 100644 .github/workflows/test_build_backend_image.yml create mode 100644 .github/workflows/test_build_frontend_image.yml delete mode 100644 .restyled.yaml delete mode 100644 checkstyle.xml delete mode 100644 heroku.yml delete mode 100644 utilities/eclipse.cleanup.xml delete mode 100644 utilities/eclipse.formatter.xml delete mode 100644 utilities/intellij.formatter.xml diff --git a/.github/workflows/test_build_backend_image.yml b/.github/workflows/test_build_backend_image.yml new file mode 100644 index 0000000000..d7991ae1e5 --- /dev/null +++ b/.github/workflows/test_build_backend_image.yml @@ -0,0 +1,45 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Test building backend image + +on: + pull_request: + types: [opened, reopened, synchronize] + branches: + - develop + - master + paths: + - 'backend/**' + - 'executable/**' + - 'Dockerfile' + - 'pom.xml' + - 'lombok.config' + - 'scripts/**' + - '.github/workflows/test_build_backend_image.yml' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}-backend + +jobs: + test-build-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + # Fetch all tags + fetch-depth: 0 + + - name: Build docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: false diff --git a/.github/workflows/test_build_frontend_image.yml b/.github/workflows/test_build_frontend_image.yml new file mode 100644 index 0000000000..53f901f2f5 --- /dev/null +++ b/.github/workflows/test_build_frontend_image.yml @@ -0,0 +1,40 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Test building frontend image + +on: + pull_request: + types: [opened, reopened, synchronize] + branches: + - develop + - master + paths: + - 'frontend/*' + - 'scripts/*' + - '.github/workflows/test_build_frontend_image.yml' +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}-frontend + +jobs: + test-build-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + # Fetch all tags + fetch-depth: 0 + + - name: Build docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: ./frontend + push: false diff --git a/.restyled.yaml b/.restyled.yaml deleted file mode 100644 index 8c27fc9961..0000000000 --- a/.restyled.yaml +++ /dev/null @@ -1,19 +0,0 @@ -enabled: false -#auto: false -#pull_requests: true -#comments: true -request_review: author -labels: ["code-style"] - -restylers: - # - astyle: - # include: - # - "**/*.java" - # arguments: - # - --style=java - # - --indent=force-tab=4 - # - --lineend=linux - # - --indent-after-parens - # - --break-closing-braces - # - --break-one-line-headers - # - --add-braces \ No newline at end of file diff --git a/checkstyle.xml b/checkstyle.xml deleted file mode 100644 index 843c435536..0000000000 --- a/checkstyle.xml +++ /dev/null @@ -1,163 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/heroku.yml b/heroku.yml deleted file mode 100644 index f38927c6d9..0000000000 --- a/heroku.yml +++ /dev/null @@ -1,4 +0,0 @@ -#deploys develop pushes to https://conquery-dev.herokuapp.com/app/static and master to https://conquery.herokuapp.com/app/static -build: - docker: - web: frontend/Dockerfile diff --git a/utilities/eclipse.cleanup.xml b/utilities/eclipse.cleanup.xml deleted file mode 100644 index ab7122abc5..0000000000 --- a/utilities/eclipse.cleanup.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/utilities/eclipse.formatter.xml b/utilities/eclipse.formatter.xml deleted file mode 100644 index 280d2afdef..0000000000 --- a/utilities/eclipse.formatter.xml +++ /dev/null @@ -1,315 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/utilities/intellij.formatter.xml b/utilities/intellij.formatter.xml deleted file mode 100644 index a106c86d86..0000000000 --- a/utilities/intellij.formatter.xml +++ /dev/null @@ -1,58 +0,0 @@ - - \ No newline at end of file From b6bb51ecb6e1ce193123932ce5e538cb1b3bf430 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 17 May 2023 13:39:15 +0200 Subject: [PATCH 34/45] adds `ConceptColumnT` to ConceptColumnSelect if asIds=true --- .../select/concept/ConceptColumnSelect.java | 16 ++++++++++++++++ .../value/ConceptElementsAggregator.java | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index e20a546551..9cdcccbd74 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept; +import java.util.Set; + +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.datasets.concepts.select.Select; @@ -7,6 +10,8 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.ConceptElementsAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.ConceptValuesAggregator; +import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; +import com.bakdata.conquery.models.types.SemanticType; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import lombok.Data; @@ -32,6 +37,17 @@ public Aggregator createAggregator() { return new ConceptValuesAggregator(((TreeConcept) getHolder().findConcept())); } + @Override + public SelectResultInfo getResultInfo(CQConcept cqConcept) { + Set additionalSemantics = null; + + if (isAsIds()) { + additionalSemantics = Set.of(new SemanticType.ConceptColumnT(cqConcept.getConcept())); + } + + return new SelectResultInfo(this, cqConcept, additionalSemantics); + } + @JsonIgnore @ValidationMethod(message = "Holder must be TreeConcept.") public boolean isHolderTreeConcept() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java index e8118b950f..311e77c384 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java @@ -49,7 +49,7 @@ public void collectRequiredTables(Set requiredTables) { @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { - Connector connector = tableConnectors.get(currentTable); + final Connector connector = tableConnectors.get(currentTable); if (connector == null) { column = null; From 8b879912335b0c61ba7cd6aa3df8c0ddf6484eed Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Wed, 17 May 2023 15:34:40 +0200 Subject: [PATCH 35/45] turn date helper functions into hooks --- frontend/src/js/common/helpers/dateHelper.ts | 10 +++--- .../ui-components/InputDate/CustomHeader.tsx | 33 ++++++++++--------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/frontend/src/js/common/helpers/dateHelper.ts b/frontend/src/js/common/helpers/dateHelper.ts index b522810f02..c5fc13fc8c 100644 --- a/frontend/src/js/common/helpers/dateHelper.ts +++ b/frontend/src/js/common/helpers/dateHelper.ts @@ -10,7 +10,6 @@ import { formatDistance, } from "date-fns"; import { de, enGB } from "date-fns/locale"; -import i18next from "i18next"; import { useTranslation } from "react-i18next"; // To save the date in this format in the state @@ -215,15 +214,16 @@ export function getFirstAndLastDateOfRange(dateRangeStr: string): { return { first, last }; } -export function getMonthName(date: Date): string { - const locale = i18next.language === "de" ? de : enGB; +export function useMonthName(date: Date): string { + const locale = useDateLocale(); return format(date, "LLLL", { locale }); } -export function getMonthNames(): string[] { +export function useMonthNames(): string[] { + const locale = useDateLocale(); return [...Array(12).keys()].map((month) => { const date = new Date(); date.setMonth(month); - return getMonthName(date); + return format(date, "LLLL", { locale }); }); } diff --git a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx index 71071a3732..1ca9009a4b 100644 --- a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx +++ b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx @@ -9,7 +9,7 @@ import { ReactDatePickerCustomHeaderProps } from "react-datepicker"; import { SelectOptionT } from "../../api/types"; import IconButton from "../../button/IconButton"; import { TransparentButton } from "../../button/TransparentButton"; -import { getMonthName, getMonthNames } from "../../common/helpers/dateHelper"; +import { useMonthName, useMonthNames } from "../../common/helpers/dateHelper"; import { List, Menu } from "../InputSelect/InputSelectComponents"; export const Root = styled("div")` @@ -36,20 +36,6 @@ export const MonthYearLabel = styled("div")` cursor: pointer; `; -const yearSelectionSpan = 10; -const yearOptions: SelectOptionT[] = [...Array(yearSelectionSpan).keys()] - .map((n) => new Date().getFullYear() - n) - .map((year) => ({ - label: String(year), - value: year, - })) - .reverse(); - -const monthOptions: SelectOptionT[] = getMonthNames().map((month, i) => ({ - label: month, - value: i, -})); - const SelectMenu = ({ date, options, @@ -87,6 +73,21 @@ const YearMonthSelect = ({ ReactDatePickerCustomHeaderProps, "date" | "changeYear" | "changeMonth" >) => { + const yearSelectionSpan = 10; + const yearOptions: SelectOptionT[] = [...Array(yearSelectionSpan).keys()] + .map((n) => new Date().getFullYear() - n) + .map((year) => ({ + label: String(year), + value: year, + })) + .reverse(); + + const monthNames = useMonthNames(); + const monthOptions: SelectOptionT[] = monthNames.map((month, i) => ({ + label: month, + value: i, + })); + const [yearSelectOpen, setYearSelectOpen] = useState(false); const [monthSelectOpen, setMonthSelectOpen] = useState(false); const handleClick = () => { @@ -101,7 +102,7 @@ const YearMonthSelect = ({ return ( <> - {getMonthName(date)} {date.getFullYear()} + {useMonthName(date)} {date.getFullYear()} {yearSelectOpen && ( Date: Mon, 22 May 2023 15:05:36 +0200 Subject: [PATCH 36/45] Also unique-ify tab field names --- frontend/src/js/external-forms/FormConfigLoader.tsx | 3 ++- frontend/src/js/external-forms/reducer.ts | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/js/external-forms/FormConfigLoader.tsx b/frontend/src/js/external-forms/FormConfigLoader.tsx index 3d75b8b77f..5d06b9f04b 100644 --- a/frontend/src/js/external-forms/FormConfigLoader.tsx +++ b/frontend/src/js/external-forms/FormConfigLoader.tsx @@ -116,6 +116,7 @@ const FormConfigLoader: FC = ({ if (!formConfig || !formConfigToLoadNext) return; const entries = Object.entries(formConfigToLoadNext.values); + const formFields = collectAllFormFields(formConfig.fields); for (const [fieldname, value] of entries) { // -------------------------- @@ -123,7 +124,7 @@ const FormConfigLoader: FC = ({ // because we changed the SELECT values: // from string, e.g. 'next' // to SelectValueT, e.g. { value: 'next', label: 'Next' } - const field = collectAllFormFields(formConfig.fields).find( + const field = formFields.find( (f): f is Field | Tabs => f.type !== "GROUP" && f.name === getUniqueFieldname(formConfig.type, fieldname), diff --git a/frontend/src/js/external-forms/reducer.ts b/frontend/src/js/external-forms/reducer.ts index e407beb3a2..bb77b5f528 100644 --- a/frontend/src/js/external-forms/reducer.ts +++ b/frontend/src/js/external-forms/reducer.ts @@ -23,6 +23,7 @@ const transformToUniqueFieldnames = ( case "TABS": return { ...field, + name: getUniqueFieldname(formType, field.name), tabs: field.tabs.map((tab) => ({ ...tab, fields: transformToUniqueFieldnames(formType, tab.fields), From dd60564b8fc2609e355f4efcb05fb05bfa80d2d8 Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Wed, 24 May 2023 12:57:42 +0200 Subject: [PATCH 37/45] set first day of the week to monday in datepicker and added hover effect for yearmonth selection --- frontend/src/js/ui-components/InputDate/CustomHeader.tsx | 6 ++++++ frontend/src/js/ui-components/InputDate/InputDate.tsx | 1 + 2 files changed, 7 insertions(+) diff --git a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx index 1ca9009a4b..ca1269e39f 100644 --- a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx +++ b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx @@ -34,6 +34,12 @@ export const OptionList = styled(List)` export const MonthYearLabel = styled("div")` font-weight: bold; cursor: pointer; + transition: opacity ${({ theme }) => theme.transitionTime}; + opacity: 0.75; + + &:hover { + opacity: 1; + } `; const SelectMenu = ({ diff --git a/frontend/src/js/ui-components/InputDate/InputDate.tsx b/frontend/src/js/ui-components/InputDate/InputDate.tsx index a3cf0aeb52..d99ec77f55 100644 --- a/frontend/src/js/ui-components/InputDate/InputDate.tsx +++ b/frontend/src/js/ui-components/InputDate/InputDate.tsx @@ -125,6 +125,7 @@ const InputDate = forwardRef( renderCustomHeader={CustomHeader} customInput={createElement(HiddenInput)} calendarContainer={StyledCalendar} + calendarStartDay={1} /> ); From 4be2eb43ddc85ec8296955d0c399cc71d20c38cd Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 29 May 2023 14:32:06 +0200 Subject: [PATCH 38/45] Fix multi-select race condition --- .../InputMultiSelect/InputMultiSelect.tsx | 35 ++++------ .../InputMultiSelect/SelectedItem.tsx | 13 ++-- .../useSyncWithValueFromAbove.ts | 65 ------------------- 3 files changed, 22 insertions(+), 91 deletions(-) delete mode 100644 frontend/src/js/ui-components/InputMultiSelect/useSyncWithValueFromAbove.ts diff --git a/frontend/src/js/ui-components/InputMultiSelect/InputMultiSelect.tsx b/frontend/src/js/ui-components/InputMultiSelect/InputMultiSelect.tsx index 5cfe3be56f..4fefe99913 100644 --- a/frontend/src/js/ui-components/InputMultiSelect/InputMultiSelect.tsx +++ b/frontend/src/js/ui-components/InputMultiSelect/InputMultiSelect.tsx @@ -38,7 +38,6 @@ import { useCloseOnClickOutside } from "./useCloseOnClickOutside"; import { useFilteredOptions } from "./useFilteredOptions"; import { useLoadMoreInitially } from "./useLoadMoreInitially"; import { useResolvableSelect } from "./useResolvableSelect"; -import { useSyncWithValueFromAbove } from "./useSyncWithValueFromAbove"; const MAX_SELECTED_ITEMS_LIMIT = 200; @@ -104,8 +103,6 @@ const InputMultiSelect = ({ const [inputValue, setInputValue] = useState(""); const { t } = useTranslation(); - const [syncingState, setSyncingState] = useState(false); - const { getSelectedItemProps, getDropdownProps, @@ -117,9 +114,9 @@ const InputMultiSelect = ({ activeIndex, } = useMultipleSelection({ initialSelectedItems: defaultValue || [], - onSelectedItemsChange: (changes) => { + selectedItems: value, + onStateChange: (changes) => { if (changes.selectedItems) { - setSyncingState(true); onChange(changes.selectedItems); } }, @@ -166,27 +163,31 @@ const InputMultiSelect = ({ case useCombobox.stateChangeTypes.InputKeyDownEnter: case useCombobox.stateChangeTypes.InputBlur: case useCombobox.stateChangeTypes.ItemClick: + // Support disabled items if (changes.selectedItem?.disabled) { return state; } + // Make sure we're staying around the index of the item that was just selected const stayAlmostAtTheSamePositionIndex = state.highlightedIndex === filteredOptions.length - 1 ? state.highlightedIndex - 1 : state.highlightedIndex; + // Determine the right item to be "chosen", supporting "creatable" items const hasChosenCreatableItem = creatable && state.highlightedIndex === 0 && inputValue.length > 0; - // The item that will be "chosen" const selectedItem = hasChosenCreatableItem ? { value: inputValue, label: inputValue } : changes.selectedItem; - if ( - selectedItem && - !selectedItems.find((item) => selectedItem.value === item.value) - ) { + const hasItemHighlighted = state.highlightedIndex > -1; + const isNotSelectedYet = + !!selectedItem && + !selectedItems.find((item) => selectedItem.value === item.value); + + if (isNotSelectedYet && hasItemHighlighted) { addSelectedItem(selectedItem); } @@ -246,14 +247,6 @@ const InputMultiSelect = ({ const clickOutsideRef = useCloseOnClickOutside({ isOpen, toggleMenu }); - useSyncWithValueFromAbove({ - value, - selectedItems, - setSelectedItems, - syncingState, - setSyncingState, - }); - const clearStaleSearch = () => { if (!isOpen) { setInputValue(""); @@ -288,12 +281,12 @@ const InputMultiSelect = ({ > - {selectedItems.map((option, index) => { + {selectedItems.map((item, index) => { return ( ( ( - { index, option, disabled, removeSelectedItem, getSelectedItemProps }, + { index, item, disabled, removeSelectedItem, getSelectedItemProps }, ref, ) => { - const label = option.selectedLabel || option.label || option.value; + const label = item.selectedLabel || item.label || item.value; const selectedItemProps = getSelectedItemProps({ - selectedItem: option, + selectedItem: item, index, }); @@ -57,7 +57,10 @@ const SelectedItem = forwardRef< removeSelectedItem(option)} + onClick={(e) => { + e.stopPropagation(); // otherwise the click handler on the Container overrides this + removeSelectedItem(item); + }} /> ); diff --git a/frontend/src/js/ui-components/InputMultiSelect/useSyncWithValueFromAbove.ts b/frontend/src/js/ui-components/InputMultiSelect/useSyncWithValueFromAbove.ts deleted file mode 100644 index 474a9a968b..0000000000 --- a/frontend/src/js/ui-components/InputMultiSelect/useSyncWithValueFromAbove.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { useEffect } from "react"; - -import type { SelectOptionT } from "../../api/types"; -import { usePrevious } from "../../common/helpers/usePrevious"; - -/** - * The idea here is that we want to allow parent components to update the `value` state - * at any time – then the selected items should update accordingly. - * - * Unfortunately, it led to an infinite loop, when selecting items fast. - * 1) every click causes an internal change to `selectedItems` - * 2) this change is synced up using `onStateChange` from `useMultipleSelection` - * 3) which triggers a change to value and causes a rerender + this effect to re-run - * 4) but this effect will already have a new state for selectedItems - * 5) so we're calling setSelectedItems with an outdated value - * - * => To counter this, we introduced a "syncing" boolean, which we set true in 2) and double-check before 5) - * - * This works, but there is probably a better way to solve this, - * like trying to tie `value` to `selectedItems` more directly - * (= having more of a "controled component state"). - */ -export const useSyncWithValueFromAbove = ({ - value, - selectedItems, - setSelectedItems, - syncingState, - setSyncingState, -}: { - value: SelectOptionT[]; - setSelectedItems: (items: SelectOptionT[]) => void; - selectedItems: SelectOptionT[]; - syncingState: boolean; - setSyncingState: (syncing: boolean) => void; -}) => { - const prevValue = usePrevious(value); - - useEffect(() => { - const prevValueStr = JSON.stringify(prevValue); - const valueStr = JSON.stringify(value); - const selectedItemsStr = JSON.stringify(selectedItems); - - const valueChanged = prevValueStr !== valueStr; - const weDontHaveValueAlreadySelected = valueStr !== selectedItemsStr; - - const takeFromAbove = valueChanged && weDontHaveValueAlreadySelected; - - if (syncingState) { - // Helps prevent race conditions, when selecting options fast - setSyncingState(false); - return; - } - - if (takeFromAbove) { - setSelectedItems(value); - } - }, [ - selectedItems, - setSelectedItems, - prevValue, - value, - syncingState, - setSyncingState, - ]); -}; From 8115c7c321e8becc4a11fc35f4cb40ce22d37b65 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 31 May 2023 16:00:26 +0200 Subject: [PATCH 39/45] Implements getValue for ExternalForm, also move the field into subclasses and make it abstract --- .../conquery/apiv1/forms/ExternalForm.java | 48 +++++++++++++------ .../bakdata/conquery/apiv1/forms/Form.java | 8 +--- .../apiv1/forms/export_form/ExportForm.java | 9 ++++ .../forms/export_form/FullExportForm.java | 10 ++++ .../FormConfigProcessor.java | 20 ++++---- .../query/preview/EntityPreviewForm.java | 2 + .../conquery/api/form/config/TestForm.java | 12 +++++ 7 files changed, 79 insertions(+), 30 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java index 0a8a8e6908..49ec7b6ca2 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java @@ -26,9 +26,9 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonValue; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.BeanProperty; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; @@ -43,7 +43,9 @@ import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; import lombok.Setter; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.Nullable; @Getter @Setter @@ -51,19 +53,23 @@ @JsonDeserialize(using = Deserializer.class) @RequiredArgsConstructor @Slf4j +@ToString public class ExternalForm extends Form implements SubTyped { @JsonValue + @ToString.Exclude private final ObjectNode node; - private final String subType; - + @Nullable @Override - public String getFormType() { - return CPSTypeIdResolver.createSubTyped(this.getClass().getAnnotation(CPSType.class).id(), getSubType()); + @JsonIgnore + public JsonNode getValues() { + return node; } + private final String subType; + @Override public String getLocalizedTypeLabel() { final JsonNode formTitle = node.get("title"); @@ -72,41 +78,52 @@ public String getLocalizedTypeLabel() { } // Form had no specific title set. Try localized lookup in FormConfig - Locale preferredLocale = I18n.LOCALE.get(); - FormType frontendConfig = FormScanner.FRONTEND_FORM_CONFIGS.get(this.getFormType()); + final Locale preferredLocale = I18n.LOCALE.get(); + final FormType frontendConfig = FormScanner.FRONTEND_FORM_CONFIGS.get(getFormType()); if (frontendConfig == null) { return getSubType(); } - JsonNode titleObj = frontendConfig.getRawConfig().path("title"); + final JsonNode titleObj = frontendConfig.getRawConfig().path("title"); if (!titleObj.isObject()) { log.trace("Expected \"title\" member to be of type Object in {}", frontendConfig); return getSubType(); } - List localesFound = new ArrayList<>(); + final List localesFound = new ArrayList<>(); titleObj.fieldNames().forEachRemaining((lang) -> localesFound.add(new Locale(lang))); + if (localesFound.isEmpty()) { log.trace("Could not extract a locale from the provided FrontendConfig: {}", frontendConfig); return getSubType(); } + Locale chosenLocale = Locale.lookup(Locale.LanguageRange.parse(preferredLocale.getLanguage()), localesFound); + if (chosenLocale == null) { chosenLocale = localesFound.get(0); log.trace("Locale lookup did not return a matching locale. Using the first title encountered: {}", chosenLocale); } - JsonNode title = titleObj.path(chosenLocale.getLanguage()); + + final JsonNode title = titleObj.path(chosenLocale.getLanguage()); + if (!title.isTextual()) { log.trace("Expected a textual node for the localized title. Was: {}", title.getNodeType()); return getSubType(); } - String ret = title.asText(); + + final String ret = title.asText(); log.trace("Extracted localized title. Was: \"{}\"", ret); return ret.isBlank() ? getSubType() : ret; } + @Override + public String getFormType() { + return CPSTypeIdResolver.createSubTyped(getClass().getAnnotation(CPSType.class).id(), getSubType()); + } + @Override public ManagedExecution toManagedExecution(User user, Dataset submittedDataset, MetaStorage storage) { return new ExternalExecution(this, user, submittedDataset, storage); @@ -142,22 +159,25 @@ public static class Deserializer extends JsonDeserializer implemen private String subTypeId; @Override - public ExternalForm deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException { + public ExternalForm deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { if (Strings.isNullOrEmpty(subTypeId)) { throw new IllegalStateException("This class needs subtype information for deserialization."); } - ObjectNode tree = p.readValueAsTree(); + + final ObjectNode tree = p.readValueAsTree(); return new ExternalForm(tree, subTypeId); } @Override public JsonDeserializer createContextual(DeserializationContext ctxt, BeanProperty property) throws JsonMappingException { // This is only called once per typeId@SubTypeId - String subTypeId = (String) ctxt.getAttribute(CPSTypeIdResolver.ATTRIBUTE_SUB_TYPE); + final String subTypeId = (String) ctxt.getAttribute(CPSTypeIdResolver.ATTRIBUTE_SUB_TYPE); + if (Strings.isNullOrEmpty(subTypeId)) { throw new IllegalStateException("This class needs subtype information for deserialization."); } + return new Deserializer(subTypeId); } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java index 451c39d333..11bcd5f3e6 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java @@ -15,9 +15,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ClassToInstanceMap; import lombok.EqualsAndHashCode; -import lombok.Getter; import lombok.NonNull; -import lombok.Setter; /** * API representation of a form query. @@ -25,15 +23,13 @@ @EqualsAndHashCode public abstract class Form implements QueryDescription { + /** * Raw form config (basically the raw format of this form), that is used by the backend at the moment to * create a {@link com.bakdata.conquery.models.forms.configs.FormConfig} upon start of this form (see {@link ManagedForm#start()}). */ @Nullable - @Getter - @Setter - @EqualsAndHashCode.Exclude - private JsonNode values; + public abstract JsonNode getValues(); @JsonIgnore public String getFormType() { diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java index 381e4bf70d..acee28d857 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java @@ -37,19 +37,28 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonManagedReference; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableList; import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; +import lombok.ToString; @Getter @Setter @CPSType(id = "EXPORT_FORM", base = QueryDescription.class) @EqualsAndHashCode(callSuper = true) +@ToString public class ExportForm extends Form implements InternalForm { + @Getter + @Setter + @EqualsAndHashCode.Exclude + private JsonNode values; + + @NotNull @JsonProperty("queryGroup") private ManagedExecutionId queryGroupId; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java index 3bfc39a382..29b20e9a82 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java @@ -33,15 +33,25 @@ import com.bakdata.conquery.models.query.Visitable; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableList; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; +import lombok.ToString; @Getter @Setter +@ToString @CPSType(id = "FULL_EXPORT_FORM", base = QueryDescription.class) public class FullExportForm extends Form implements InternalForm { + @Getter + @Setter + @EqualsAndHashCode.Exclude + private JsonNode values; + + @NotNull @JsonProperty("queryGroup") private ManagedExecutionId queryGroupId; diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java index dae18e27c1..2356d2811c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java @@ -72,7 +72,7 @@ public Stream getConfigsByFormType(@NonNull Su // If no specific form type is provided, show all types the subject is permitted to create. // However if a subject queries for specific form types, we will show all matching regardless whether // the form config can be used by the subject again. - Set allowedFormTypes = new HashSet<>(); + final Set allowedFormTypes = new HashSet<>(); for (FormType formType : FormScanner.FRONTEND_FORM_CONFIGS.values()) { if (!subject.isPermitted(formType, Ability.CREATE)) { @@ -86,10 +86,10 @@ public Stream getConfigsByFormType(@NonNull Su final Set formTypesFinal = requestedFormType; - Stream stream = storage.getAllFormConfigs().stream() - .filter(c -> dataset.equals(c.getDataset())) - .filter(c -> formTypesFinal.contains(c.getFormType())) - .filter(c -> subject.isPermitted(c, Ability.READ)); + final Stream stream = storage.getAllFormConfigs().stream() + .filter(c -> dataset.equals(c.getDataset())) + .filter(c -> formTypesFinal.contains(c.getFormType())) + .filter(c -> subject.isPermitted(c, Ability.READ)); return stream.map(c -> c.overview(subject)); @@ -117,7 +117,7 @@ public FormConfig addConfig(Subject subject, Dataset targetDataset, FormConfigAP subject.authorize(namespace.getDataset(), Ability.READ); - FormConfig internalConfig = config.intern(storage.getUser(subject.getId()), targetDataset); + final FormConfig internalConfig = config.intern(storage.getUser(subject.getId()), targetDataset); // Add the config immediately to the submitted dataset addConfigToDataset(internalConfig); @@ -151,14 +151,14 @@ public FormConfigFullRepresentation patchConfig(Subject subject, FormConfig conf * Deletes a configuration from the storage and all permissions, that have this configuration as target. */ public void deleteConfig(Subject subject, FormConfig config) { - User user = storage.getUser(subject.getId()); + final User user = storage.getUser(subject.getId()); user.authorize(config, Ability.DELETE); storage.removeFormConfig(config.getId()); // Delete corresponding permissions (Maybe better to put it into a slow job) for (ConqueryPermission permission : user.getPermissions()) { - WildcardPermission wpermission = (WildcardPermission) permission; + final WildcardPermission wpermission = (WildcardPermission) permission; if (!wpermission.getDomains().contains(FormConfigPermission.DOMAIN.toLowerCase())) { continue; @@ -169,9 +169,9 @@ public void deleteConfig(Subject subject, FormConfig config) { if (!wpermission.getInstances().isEmpty()) { // Create new permission if it was a composite permission - Set instancesCleared = new HashSet<>(wpermission.getInstances()); + final Set instancesCleared = new HashSet<>(wpermission.getInstances()); instancesCleared.remove(config.getId().toString()); - WildcardPermission clearedPermission = + final WildcardPermission clearedPermission = new WildcardPermission(List.of(wpermission.getDomains(), wpermission.getAbilities(), instancesCleared), Instant.now()); user.addPermission(clearedPermission); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java index 61aa8a637a..9182dc9ab0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java @@ -43,6 +43,7 @@ import lombok.Getter; import lombok.NonNull; import lombok.RequiredArgsConstructor; +import lombok.ToString; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -60,6 +61,7 @@ @CPSType(id = "ENTITY_PREVIEW", base = QueryDescription.class) @Getter @RequiredArgsConstructor(onConstructor_ = {@JsonCreator}) +@ToString public class EntityPreviewForm extends Form implements InternalForm { public static final String INFOS_QUERY_NAME = "INFOS"; diff --git a/backend/src/test/java/com/bakdata/conquery/api/form/config/TestForm.java b/backend/src/test/java/com/bakdata/conquery/api/form/config/TestForm.java index 97f7d36c88..5af22ad1e6 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/form/config/TestForm.java +++ b/backend/src/test/java/com/bakdata/conquery/api/form/config/TestForm.java @@ -18,6 +18,8 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; +import com.fasterxml.jackson.databind.JsonNode; +import org.jetbrains.annotations.Nullable; public abstract class TestForm extends Form implements InternalForm { @@ -53,9 +55,19 @@ public void visit(Consumer visitor) { @CPSType(id = "TEST_FORM_ABS_URL", base = QueryDescription.class) public static class Abs extends TestForm { + @Nullable + @Override + public JsonNode getValues() { + return null; + } } @CPSType(id = "TEST_FORM_REL_URL", base = QueryDescription.class) public static class Rel extends TestForm { + @Nullable + @Override + public JsonNode getValues() { + return null; + } } } From 52c9517ad6a8ad7485c647483663c815ab4e7446 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 31 May 2023 17:14:08 +0200 Subject: [PATCH 40/45] allows not setting queryGroup which will then select ALL entities (CQYES) --- .../apiv1/forms/export_form/AbsoluteMode.java | 8 ++--- .../apiv1/forms/export_form/ExportForm.java | 19 ++++++++++-- .../forms/export_form/FullExportForm.java | 26 +++++++++++++--- .../bakdata/conquery/apiv1/query/CQYes.java | 31 +++++++++++++++++++ 4 files changed, 71 insertions(+), 13 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java index c71dbd788c..2f58701d2e 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java @@ -46,12 +46,8 @@ public Query createSpecializedQuery() { List resolutionsAndAlignments = ExportForm.getResolutionAlignmentMap(getForm().getResolvedResolutions(), getAlignmentHint()); - return new AbsoluteFormQuery( - getForm().getPrerequisite(), - dateRange, - resolvedFeatures, - resolutionsAndAlignments - ); + Query prerequisite = getForm().getPrerequisite(); + return new AbsoluteFormQuery(prerequisite, dateRange, resolvedFeatures, resolutionsAndAlignments); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java index acee28d857..63f37f96b8 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java @@ -1,11 +1,13 @@ package com.bakdata.conquery.apiv1.forms.export_form; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; +import javax.annotation.Nullable; import javax.validation.Valid; import javax.validation.ValidationException; import javax.validation.constraints.NotEmpty; @@ -16,6 +18,8 @@ import com.bakdata.conquery.apiv1.forms.Form; import com.bakdata.conquery.apiv1.forms.InternalForm; import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.apiv1.query.CQYes; +import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.internationalization.ExportFormC10n; @@ -59,7 +63,7 @@ public class ExportForm extends Form implements InternalForm { private JsonNode values; - @NotNull + @Nullable @JsonProperty("queryGroup") private ManagedExecutionId queryGroupId; @@ -106,19 +110,28 @@ public Map createSubQueries() { @Override public Set collectRequiredQueries() { + if (queryGroupId == null) { + return Collections.emptySet(); + } + return Set.of(queryGroupId); } @Override public void resolve(QueryResolveContext context) { - queryGroup = (ManagedQuery) context.getStorage().getExecution(queryGroupId); + if(queryGroupId != null) { + queryGroup = (ManagedQuery) context.getStorage().getExecution(queryGroupId); + prerequisite = queryGroup.getQuery(); + } + else { + prerequisite = new ConceptQuery(new CQYes()); + } // Apply defaults to user concept ExportForm.DefaultSelectSettable.enable(features); timeMode.resolve(context); - prerequisite = queryGroup.getQuery(); List resolutionsFlat = resolution.stream() .flatMap(ResolutionShortNames::correspondingResolutions) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java index 29b20e9a82..f457c1a04f 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.apiv1.forms.export_form; import java.time.LocalDate; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -9,12 +10,13 @@ import javax.annotation.Nullable; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; -import javax.validation.constraints.NotNull; import c10n.C10N; import com.bakdata.conquery.ConqueryConstants; import com.bakdata.conquery.apiv1.forms.Form; import com.bakdata.conquery.apiv1.forms.InternalForm; +import com.bakdata.conquery.apiv1.query.CQYes; +import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.TableExportQuery; @@ -46,13 +48,14 @@ @CPSType(id = "FULL_EXPORT_FORM", base = QueryDescription.class) public class FullExportForm extends Form implements InternalForm { + @Nullable @Getter @Setter @EqualsAndHashCode.Exclude private JsonNode values; - @NotNull + @Nullable @JsonProperty("queryGroup") private ManagedExecutionId queryGroupId; @@ -78,7 +81,16 @@ public Map createSubQueries() { // Forms are sent as an array of standard queries containing AND/OR of CQConcepts, we ignore everything and just convert the CQConcepts into CQUnfiltered for export. - final TableExportQuery exportQuery = new TableExportQuery(queryGroup.getQuery()); + final Query query; + + if (queryGroupId != null) { + query = queryGroup.getQuery(); + } + else { + query = new ConceptQuery(new CQYes()); + } + + final TableExportQuery exportQuery = new TableExportQuery(query); exportQuery.setDateRange(getDateRange()); exportQuery.setTables(tables); @@ -93,12 +105,18 @@ public Map createSubQueries() { @Override public Set collectRequiredQueries() { + if (queryGroupId == null) { + return Collections.emptySet(); + } + return Set.of(queryGroupId); } @Override public void resolve(QueryResolveContext context) { - queryGroup = (ManagedQuery) context.getStorage().getExecution(queryGroupId); + if (queryGroupId != null) { + queryGroup = (ManagedQuery) context.getStorage().getExecution(queryGroupId); + } } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java new file mode 100644 index 0000000000..03aab80a99 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java @@ -0,0 +1,31 @@ +package com.bakdata.conquery.apiv1.query; + +import java.util.Collections; +import java.util.List; + +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.query.QueryPlanContext; +import com.bakdata.conquery.models.query.QueryResolveContext; +import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; +import com.bakdata.conquery.models.query.queryplan.QPNode; +import com.bakdata.conquery.models.query.queryplan.specific.Leaf; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; + +@CPSType(id = "YES", base = CQElement.class) +public class CQYes extends CQElement{ + + @Override + public void resolve(QueryResolveContext context) { + + } + + @Override + public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { + return new Leaf(); + } + + @Override + public List getResultInfos() { + return Collections.emptyList(); + } +} From cef7ec5abe7f2747475d0c0b576cbed037f789db Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 1 Jun 2023 10:01:34 +0200 Subject: [PATCH 41/45] adds test for CQYes and implements YesNode, as it needs ALL_IDS_TABLE as marker --- .../bakdata/conquery/apiv1/query/CQYes.java | 13 ++++- .../models/query/queryplan/specific/Yes.java | 57 +++++++++++++++++++ .../tests/query/CQYES/CQYES.test.json | 32 +++++++++++ .../resources/tests/query/CQYES/content.csv | 7 +++ .../resources/tests/query/CQYES/expected.csv | 7 +++ 5 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java create mode 100644 backend/src/test/resources/tests/query/CQYES/CQYES.test.json create mode 100644 backend/src/test/resources/tests/query/CQYES/content.csv create mode 100644 backend/src/test/resources/tests/query/CQYES/expected.csv diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java index 03aab80a99..0031abe6bc 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java @@ -4,15 +4,17 @@ import java.util.List; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; +import com.bakdata.conquery.models.query.RequiredEntities; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; import com.bakdata.conquery.models.query.queryplan.QPNode; -import com.bakdata.conquery.models.query.queryplan.specific.Leaf; +import com.bakdata.conquery.models.query.queryplan.specific.Yes; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; @CPSType(id = "YES", base = CQElement.class) -public class CQYes extends CQElement{ +public class CQYes extends CQElement { @Override public void resolve(QueryResolveContext context) { @@ -21,11 +23,16 @@ public void resolve(QueryResolveContext context) { @Override public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { - return new Leaf(); + return new Yes(context.getDataset().getAllIdsTable()); } @Override public List getResultInfos() { return Collections.emptyList(); } + + @Override + public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { + return new RequiredEntities(context.getBucketManager().getEntities().keySet()); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java new file mode 100644 index 0000000000..fd2c228a9b --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java @@ -0,0 +1,57 @@ +package com.bakdata.conquery.models.query.queryplan.specific; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.events.Bucket; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; +import com.bakdata.conquery.models.query.queryplan.QPNode; +import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; +import lombok.Data; +import lombok.ToString; + +@ToString +@Data +public class Yes extends QPNode { + + private final Table table; + + @Override + public void init(Entity entity, QueryExecutionContext context) { + super.init(entity, context); + } + + @Override + public void acceptEvent(Bucket bucket, int event) { + + } + + @Override + public void collectRequiredTables(Set
requiredTables) { + requiredTables.add(table); + } + + @Override + public boolean isContained() { + return true; + } + + @Override + public Collection> getDateAggregators() { + return Collections.emptySet(); + } + + @Override + public boolean isOfInterest(Bucket bucket) { + return true; + } + + @Override + public boolean isOfInterest(Entity entity) { + return true; + } +} diff --git a/backend/src/test/resources/tests/query/CQYES/CQYES.test.json b/backend/src/test/resources/tests/query/CQYES/CQYES.test.json new file mode 100644 index 0000000000..23aafee084 --- /dev/null +++ b/backend/src/test/resources/tests/query/CQYES/CQYES.test.json @@ -0,0 +1,32 @@ +{ + "type": "QUERY_TEST", + "label": "CQYES Test", + "expectedCsv": "tests/query/CQYES/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type" : "YES" + } + }, + "concepts": [ + + ], + "content": { + "tables": [ + { + "csv": "tests/query/CQYES/content.csv", + "name": "test_table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/query/CQYES/content.csv b/backend/src/test/resources/tests/query/CQYES/content.csv new file mode 100644 index 0000000000..5499484b3d --- /dev/null +++ b/backend/src/test/resources/tests/query/CQYES/content.csv @@ -0,0 +1,7 @@ +pid,datum +4,2013-11-10 +5,2013-11-10 +6,2013-11-10 +1,2012-01-01 +2,2013-11-10 +3,2013-11-10 diff --git a/backend/src/test/resources/tests/query/CQYES/expected.csv b/backend/src/test/resources/tests/query/CQYES/expected.csv new file mode 100644 index 0000000000..a8bf68f3a1 --- /dev/null +++ b/backend/src/test/resources/tests/query/CQYES/expected.csv @@ -0,0 +1,7 @@ +result,dates +4,{} +5,{} +6,{} +1,{} +2,{} +3,{} From 8b6c758bec9ee1173dd8c4861d7e31b5ea84bdf0 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:12:33 +0200 Subject: [PATCH 42/45] fixes additionalSemantics being null --- .../datasets/concepts/select/concept/ConceptColumnSelect.java | 3 ++- .../conquery/models/query/resultinfo/SelectResultInfo.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index 9cdcccbd74..f02d2fb086 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept; +import java.util.Collections; import java.util.Set; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -39,7 +40,7 @@ public Aggregator createAggregator() { @Override public SelectResultInfo getResultInfo(CQConcept cqConcept) { - Set additionalSemantics = null; + Set additionalSemantics = Collections.emptySet(); if (isAsIds()) { additionalSemantics = Set.of(new SemanticType.ConceptColumnT(cqConcept.getConcept())); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java index f427fbdaeb..9b993666b5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java @@ -26,6 +26,7 @@ public class SelectResultInfo extends ResultInfo { private final CQConcept cqConcept; @Getter(AccessLevel.PACKAGE) + @NonNull private final Set additionalSemantics; public SelectResultInfo(Select select, CQConcept cqConcept) { From 5d230eef0df42c90c7370432705208794cfa39a1 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:38:00 +0200 Subject: [PATCH 43/45] removes NOT_EMPTY validation for frontend, as no query-group is legal --- .../forms/export_form.frontend_conf.json | 653 +++++++++--------- .../table_export_form.frontend_conf.json | 173 +++-- 2 files changed, 412 insertions(+), 414 deletions(-) diff --git a/backend/src/main/resources/com/bakdata/conquery/frontend/forms/export_form.frontend_conf.json b/backend/src/main/resources/com/bakdata/conquery/frontend/forms/export_form.frontend_conf.json index 00c675d710..8516063810 100644 --- a/backend/src/main/resources/com/bakdata/conquery/frontend/forms/export_form.frontend_conf.json +++ b/backend/src/main/resources/com/bakdata/conquery/frontend/forms/export_form.frontend_conf.json @@ -1,351 +1,350 @@ { - "title": { - "en": "Data Export", - "de": "Datenexport" + "title": { + "en": "Data Export", + "de": "Datenexport" + }, + "description": { + "de": "Mit diesem Formular werden Konzept- und Ausgabewerte für jeden Versicherten einer Anfrage einzeln auf einen Beobachtungszeitraum aggregiert. Zusätzlich zum gesamten Zeitraum kann dieser nochmal in Jahre oder Quartale unterteilt werden. Die Daten können dabei in einem absoluten Beobachtungszeitraum oder relativ zu einem mit der Anfrage erstellten Indexdatum analysiert werden. Die Ausgabe kann sowohl als Excel, als auch CSV heruntergeladen werden.", + "en": "With this form, concept and output values for each insured person of a query are aggregated individually to an observation period. In addition to the entire period, this can be subdivided again into years or quarters. The data can be analyzed in an absolute observation period or relative to an index date created with the query. The output can be downloaded as Excel as well as CSV." + }, + "type": "EXPORT_FORM", + "fields": [ + { + "label": { + "en": "Cohort", + "de": "Versichertengruppe" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" }, - "description": { - "de": "Mit diesem Formular werden Konzept- und Ausgabewerte für jeden Versicherten einer Anfrage einzeln auf einen Beobachtungszeitraum aggregiert. Zusätzlich zum gesamten Zeitraum kann dieser nochmal in Jahre oder Quartale unterteilt werden. Die Daten können dabei in einem absoluten Beobachtungszeitraum oder relativ zu einem mit der Anfrage erstellten Indexdatum analysiert werden. Die Ausgabe kann sowohl als Excel, als auch CSV heruntergeladen werden.", - "en": "With this form, concept and output values for each insured person of a query are aggregated individually to an observation period. In addition to the entire period, this can be subdivided again into years or quarters. The data can be analyzed in an absolute observation period or relative to an index date created with the query. The output can be downloaded as Excel as well as CSV." - }, - "type": "EXPORT_FORM", - "fields": [ - { - "label": { - "en": "Cohort", - "de": "Versichertengruppe" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, + { + "name": "queryGroup", + "type": "RESULT_GROUP", + "label": { + "de": "Versichertengruppe (Anfrage)", + "en": "Cohort (Previous Query)" + }, + "dropzoneLabel": { + "de": "Füge eine Versichertengruppe aus einer bestehenden Anfrage hinzu", + "en": "Add a cohort from a previous query" + }, + "validations": [ + ], + "tooltip": { + "de": "Versichertengruppe (Anfrage) für die Daten ausgegeben werden soll.", + "en": "Cohort whose data is exported" + } + }, + { + "label": { + "de": "Zeitlicher Bezug", + "en": "Time Reference" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" + }, + { + "name": "timeMode", + "type": "TABS", + "defaultValue": "ABSOLUTE", + "tabs": [ { - "name": "queryGroup", - "type": "RESULT_GROUP", - "label": { - "de": "Versichertengruppe (Anfrage)", - "en": "Cohort (Previous Query)" - }, - "dropzoneLabel": { - "de": "Füge eine Versichertengruppe aus einer bestehenden Anfrage hinzu", - "en": "Add a cohort from a previous query" - }, - "validations": [ + "name": "ABSOLUTE", + "title": { + "de": "Absolut", + "en": "Absolute" + }, + "fields": [ + { + "name": "dateRange", + "type": "DATE_RANGE", + "label": { + "de": "Beobachtungszeitraum", + "en": "Observation Period" + }, + "validations": [ "NOT_EMPTY" - ], - "tooltip": { - "de": "Versichertengruppe (Anfrage) für die Daten ausgegeben werden soll.", - "en": "Cohort whose data is exported" - } + ] + } + ], + "tooltip": { + "de": "Die Ausgaben beziehen sich auf einen festen absoluten Zeitraum.", + "en": "The output relates to a fixed absolute period." + } }, { - "label": { - "de": "Zeitlicher Bezug", - "en": "Time Reference" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, - { - "name": "timeMode", - "type": "TABS", - "defaultValue": "ABSOLUTE", - "tabs": [ + "name": "RELATIVE", + "title": { + "de": "Relativ", + "en": "Relative" + }, + "fields": [ + { + "name": "timeUnit", + "type": "SELECT", + "label": { + "de": "Zeiteinheit des Vor- und Nachbeobachtungszeitraums", + "en": "Time unit of feature and outcome periods" + }, + "defaultValue": "QUARTERS", + "options": [ { - "name": "ABSOLUTE", - "title": { - "de": "Absolut", - "en": "Absolute" - }, - "fields": [ - { - "name": "dateRange", - "type": "DATE_RANGE", - "label": { - "de": "Beobachtungszeitraum", - "en": "Observation Period" - }, - "validations": [ - "NOT_EMPTY" - ] - } - ], - "tooltip": { - "de": "Die Ausgaben beziehen sich auf einen festen absoluten Zeitraum.", - "en": "The output relates to a fixed absolute period." - } + "label": { + "de": "Tage", + "en": "Days" + }, + "value": "DAYS" }, { - "name": "RELATIVE", - "title": { - "de": "Relativ", - "en": "Relative" - }, - "fields": [ - { - "name": "timeUnit", - "type": "SELECT", - "label": { - "de": "Zeiteinheit des Vor- und Nachbeobachtungszeitraums", - "en": "Time unit of feature and outcome periods" - }, - "defaultValue": "QUARTERS", - "options": [ - { - "label": { - "de": "Tage", - "en": "Days" - }, - "value": "DAYS" - }, - { - "label": { - "de": "Quartale", - "en": "Quarters" - }, - "value": "QUARTERS" - } - ], - "validations": [ - "NOT_EMPTY" - ], - "tooltip": { - "de": "Die Zeiteinheit bezieht sich auf die folgenden Eingabefelder, welche den Zeitraum vor und nach dem Indexdatum bestimmen.", - "en": "The time unit refers to the following input fields, which determine the period before and after the index date." - } - }, - { - "name": "timeCountBefore", - "type": "NUMBER", - "defaultValue": 4, - "min": 1, - "label": { - "de": "Zeit davor", - "en": "Units before" - }, - "placeholder": { - "de": "4", - "en": "4" - }, - "pattern": "^(?!-)\\d*$", - "validations": [ - "NOT_EMPTY", - "GREATER_THAN_ZERO" - ], - "tooltip": { - "de": "Anzahl an Zeiteinheiten, die die Größe des Zeitraums vor dem Indexdatum bestimmten.", - "en": "Number of time units that determined the size of the period before the index date." - } - }, - { - "name": "timeCountAfter", - "type": "NUMBER", - "min": 1, - "defaultValue": 4, - "label": { - "de": "Zeit danach", - "en": "Units after" - }, - "placeholder": { - "de": "4", - "en": "4" - }, - "pattern": "^(?!-)\\d*$", - "validations": [ - "NOT_EMPTY", - "GREATER_THAN_ZERO" - ], - "tooltip": { - "de": "Anzahl an Zeiteinheiten, die die Größe des Zeitraums nach dem Indexdatum bestimmten.", - "en": "Number of time units that determined the size of the period after the index date." - } - }, - { - "name": "indexSelector", - "type": "SELECT", - "label": { - "de": "Zeitstempel Indexdatum", - "en": "Index date sampler" - }, - "defaultValue": "EARLIEST", - "options": [ - { - "label": { - "de": "ERSTES", - "en": "First" - }, - "value": "EARLIEST" - }, - { - "label": { - "de": "LETZTES", - "en": "Last" - }, - "value": "LATEST" - }, - { - "label": { - "de": "ZUFÄLLIG", - "en": "Random" - }, - "value": "RANDOM" - } - ], - "validations": [ - "NOT_EMPTY" - ], - "tooltip": { - "de": "Wenn mehr als ein Datumswert pro Person vorliegt, kann hier ausgewählt werden welcher als Indexdatum gewertet werden soll.", - "en": "If there is more than one date value per person, you can select here which one should be evaluated as index date." - } - }, - { - "name": "indexPlacement", - "type": "SELECT", - "label": { - "de": "Zugehörigkeit Indexdatum", - "en": "Index period inclusion" - }, - "defaultValue": "AFTER", - "options": [ - { - "label": { - "de": "VORBEOBACHTUNGSZEITRAUM", - "en": "Feature period" - }, - "value": "BEFORE" - }, - { - "label": { - "de": "NEUTRAL", - "en": "Neutral" - }, - "value": "NEUTRAL" - }, - { - "label": { - "de": "NACHBEOBACHTUNGSZEITRAUM", - "en": "Outcome period" - }, - "value": "AFTER" - } - ], - "validations": [ - "NOT_EMPTY" - ], - "tooltip": { - "de": "Angabe für welchen Zeitraum das Quartal mit dem Indexdatum gewertet wird.", - "en": "Indication which period includes the index period" - } - } - ], - "tooltip": { - "de": "Die Ausgaben beziehen sich auf einen Vor- und Nachbeobachtungszeitraum, abhängig von dem Indexdatum jeder Person in der Versichertengruppe.", - "en": "Outputs are for a pre- and post-observation period, depending on the index period of each person in the cohort." - } + "label": { + "de": "Quartale", + "en": "Quarters" + }, + "value": "QUARTERS" } - ] - }, - { - "label": { - "de": "Datengrundlage und Konzepte", - "en": "Attributes" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, - { - "name": "features", - "type": "CONCEPT_LIST", - "label": { - "de": "Konzepte", - "en": "Concepts" - }, - "isTwoDimensional": true, - "conceptDropzoneLabel": { - "de": "Füge ein Konzept oder eine Konzeptliste hinzu", - "en": "Add a concept or a concept list" - }, - "validations": [ - "NOT_EMPTY" - ] - }, - { - "label": { - "de": "Analyse und Ausgabe", - "en": "Analysis and Output" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, - { - "name": "resolution", - "type": "SELECT", - "label": { - "de": "Stratifizierung Beobachtungszeitraum", - "en": "Temporal stratification" - }, - "defaultValue": "COMPLETE", - "options": [ - { - "label": { - "de": "Gesamter Zeitraum", - "en": "Total" - }, - "value": "COMPLETE" - }, + ], + "validations": [ + "NOT_EMPTY" + ], + "tooltip": { + "de": "Die Zeiteinheit bezieht sich auf die folgenden Eingabefelder, welche den Zeitraum vor und nach dem Indexdatum bestimmen.", + "en": "The time unit refers to the following input fields, which determine the period before and after the index date." + } + }, + { + "name": "timeCountBefore", + "type": "NUMBER", + "defaultValue": 4, + "min": 1, + "label": { + "de": "Zeit davor", + "en": "Units before" + }, + "placeholder": { + "de": "4", + "en": "4" + }, + "pattern": "^(?!-)\\d*$", + "validations": [ + "NOT_EMPTY", + "GREATER_THAN_ZERO" + ], + "tooltip": { + "de": "Anzahl an Zeiteinheiten, die die Größe des Zeitraums vor dem Indexdatum bestimmten.", + "en": "Number of time units that determined the size of the period before the index date." + } + }, + { + "name": "timeCountAfter", + "type": "NUMBER", + "min": 1, + "defaultValue": 4, + "label": { + "de": "Zeit danach", + "en": "Units after" + }, + "placeholder": { + "de": "4", + "en": "4" + }, + "pattern": "^(?!-)\\d*$", + "validations": [ + "NOT_EMPTY", + "GREATER_THAN_ZERO" + ], + "tooltip": { + "de": "Anzahl an Zeiteinheiten, die die Größe des Zeitraums nach dem Indexdatum bestimmten.", + "en": "Number of time units that determined the size of the period after the index date." + } + }, + { + "name": "indexSelector", + "type": "SELECT", + "label": { + "de": "Zeitstempel Indexdatum", + "en": "Index date sampler" + }, + "defaultValue": "EARLIEST", + "options": [ { - "label": { - "de": "Jahre", - "en": "Years" - }, - "value": "YEARS" + "label": { + "de": "ERSTES", + "en": "First" + }, + "value": "EARLIEST" }, { - "label": { - "de": "Quartale", - "en": "Quarters" - }, - "value": "QUARTERS" + "label": { + "de": "LETZTES", + "en": "Last" + }, + "value": "LATEST" }, { - "label": { - "de": "Jahre und Quartale", - "en": "Years and Quarters" - }, - "value": "YEARS_QUARTERS" - }, + "label": { + "de": "ZUFÄLLIG", + "en": "Random" + }, + "value": "RANDOM" + } + ], + "validations": [ + "NOT_EMPTY" + ], + "tooltip": { + "de": "Wenn mehr als ein Datumswert pro Person vorliegt, kann hier ausgewählt werden welcher als Indexdatum gewertet werden soll.", + "en": "If there is more than one date value per person, you can select here which one should be evaluated as index date." + } + }, + { + "name": "indexPlacement", + "type": "SELECT", + "label": { + "de": "Zugehörigkeit Indexdatum", + "en": "Index period inclusion" + }, + "defaultValue": "AFTER", + "options": [ { - "label": { - "de": "Gesamter Zeitraum und Jahre", - "en": "Total and Years" - }, - "value": "COMPLETE_YEARS" + "label": { + "de": "VORBEOBACHTUNGSZEITRAUM", + "en": "Feature period" + }, + "value": "BEFORE" }, { - "label": { - "de": "Gesamter Zeitraum und Quartale", - "en": "Total and Quarters" - }, - "value": "COMPLETE_QUARTERS" + "label": { + "de": "NEUTRAL", + "en": "Neutral" + }, + "value": "NEUTRAL" }, { - "label": { - "de": "Gesamter Zeitraum, Jahre und Quartale", - "en": "Total, Years and Quarters" - }, - "value": "COMPLETE_YEARS_QUARTERS" + "label": { + "de": "NACHBEOBACHTUNGSZEITRAUM", + "en": "Outcome period" + }, + "value": "AFTER" } - ], - "validations": [ + ], + "validations": [ "NOT_EMPTY" - ] + ], + "tooltip": { + "de": "Angabe für welchen Zeitraum das Quartal mit dem Indexdatum gewertet wird.", + "en": "Indication which period includes the index period" + } + } + ], + "tooltip": { + "de": "Die Ausgaben beziehen sich auf einen Vor- und Nachbeobachtungszeitraum, abhängig von dem Indexdatum jeder Person in der Versichertengruppe.", + "en": "Outputs are for a pre- and post-observation period, depending on the index period of each person in the cohort." + } + } + ] + }, + { + "label": { + "de": "Datengrundlage und Konzepte", + "en": "Attributes" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" + }, + { + "name": "features", + "type": "CONCEPT_LIST", + "label": { + "de": "Konzepte", + "en": "Concepts" + }, + "isTwoDimensional": true, + "conceptDropzoneLabel": { + "de": "Füge ein Konzept oder eine Konzeptliste hinzu", + "en": "Add a concept or a concept list" + }, + "validations": [ + "NOT_EMPTY" + ] + }, + { + "label": { + "de": "Analyse und Ausgabe", + "en": "Analysis and Output" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" + }, + { + "name": "resolution", + "type": "SELECT", + "label": { + "de": "Stratifizierung Beobachtungszeitraum", + "en": "Temporal stratification" + }, + "defaultValue": "COMPLETE", + "options": [ + { + "label": { + "de": "Gesamter Zeitraum", + "en": "Total" + }, + "value": "COMPLETE" + }, + { + "label": { + "de": "Jahre", + "en": "Years" + }, + "value": "YEARS" + }, + { + "label": { + "de": "Quartale", + "en": "Quarters" + }, + "value": "QUARTERS" + }, + { + "label": { + "de": "Jahre und Quartale", + "en": "Years and Quarters" + }, + "value": "YEARS_QUARTERS" + }, + { + "label": { + "de": "Gesamter Zeitraum und Jahre", + "en": "Total and Years" + }, + "value": "COMPLETE_YEARS" + }, + { + "label": { + "de": "Gesamter Zeitraum und Quartale", + "en": "Total and Quarters" + }, + "value": "COMPLETE_QUARTERS" + }, + { + "label": { + "de": "Gesamter Zeitraum, Jahre und Quartale", + "en": "Total, Years and Quarters" + }, + "value": "COMPLETE_YEARS_QUARTERS" } - ] + ], + "validations": [ + "NOT_EMPTY" + ] + } + ] } \ No newline at end of file diff --git a/backend/src/main/resources/com/bakdata/conquery/frontend/forms/table_export_form.frontend_conf.json b/backend/src/main/resources/com/bakdata/conquery/frontend/forms/table_export_form.frontend_conf.json index 9569be7550..e736367a43 100644 --- a/backend/src/main/resources/com/bakdata/conquery/frontend/forms/table_export_form.frontend_conf.json +++ b/backend/src/main/resources/com/bakdata/conquery/frontend/forms/table_export_form.frontend_conf.json @@ -1,89 +1,88 @@ { - "title": { - "de": "Rohdatenexport", - "en": "Table Export" - }, - "description": { - "en": "This form is used to output the raw tables behind the specified concepts, which are linked to the concepts. The data can be analyzed in an absolute observation period.", - "de": "Mit diesem Formular werden zu den angegebenen Konzepten die dahinterliegenden, rohen Tabellen ausgegeben, welche mit den Konzepten verknüpft sind. Die Daten können dabei in einem absoluten Beobachtungszeitraum analysiert werden. Die Ausgabe kann sowohl als Excel, als auch CSV heruntergeladen werden." - }, - "type": "FULL_EXPORT_FORM", - "fields": [ - { - "label": { - "en": "Cohort", - "de": "Versichertengruppe" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, - { - "name": "queryGroup", - "type": "RESULT_GROUP", - "label": { - "de": "Versichertengruppe (Anfrage)", - "en": "Cohort (Previous Query)" - }, - "dropzoneLabel": { - "de": "Füge eine Versichertengruppe aus einer bestehenden Anfrage hinzu.", - "en": "Add a cohort from a previous query" - }, - "validations": [ - "NOT_EMPTY" - ], - "tooltip": { - "de": "Versichertengruppe (Anfrage) für die Daten ausgegeben werden soll.", - "en": "Cohort whose data is exported" - } - }, - { - "label": { - "de": "Zeitlicher Bezug", - "en": "Time Reference" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, - { - "name": "dateRange", - "type": "DATE_RANGE", - "label": { - "de": "Beobachtungszeitraum", - "en": "Observation Period" - }, - "validations": [ - "NOT_EMPTY" - ] - }, - { - "label": { - "de": "Datengrundlage und Konzepte", - "en": "Attributes" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, - { - "name": "tables", - "type": "CONCEPT_LIST", - "label": { - "de": "Konzepte", - "en": "Concepts" - }, - "isTwoDimensional": false, - "conceptDropzoneLabel": { - "en": "Add a concept or a concept list", - "de": "Füge ein Konzept oder eine Konzeptliste hinzu" - }, - "validations": [ - "NOT_EMPTY" - ] - } - ] + "title": { + "de": "Rohdatenexport", + "en": "Table Export" + }, + "description": { + "en": "This form is used to output the raw tables behind the specified concepts, which are linked to the concepts. The data can be analyzed in an absolute observation period.", + "de": "Mit diesem Formular werden zu den angegebenen Konzepten die dahinterliegenden, rohen Tabellen ausgegeben, welche mit den Konzepten verknüpft sind. Die Daten können dabei in einem absoluten Beobachtungszeitraum analysiert werden. Die Ausgabe kann sowohl als Excel, als auch CSV heruntergeladen werden." + }, + "type": "FULL_EXPORT_FORM", + "fields": [ + { + "label": { + "en": "Cohort", + "de": "Versichertengruppe" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" + }, + { + "name": "queryGroup", + "type": "RESULT_GROUP", + "label": { + "de": "Versichertengruppe (Anfrage)", + "en": "Cohort (Previous Query)" + }, + "dropzoneLabel": { + "de": "Füge eine Versichertengruppe aus einer bestehenden Anfrage hinzu.", + "en": "Add a cohort from a previous query" + }, + "validations": [ + ], + "tooltip": { + "de": "Versichertengruppe (Anfrage) für die Daten ausgegeben werden soll.", + "en": "Cohort whose data is exported" + } + }, + { + "label": { + "de": "Zeitlicher Bezug", + "en": "Time Reference" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" + }, + { + "name": "dateRange", + "type": "DATE_RANGE", + "label": { + "de": "Beobachtungszeitraum", + "en": "Observation Period" + }, + "validations": [ + "NOT_EMPTY" + ] + }, + { + "label": { + "de": "Datengrundlage und Konzepte", + "en": "Attributes" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" + }, + { + "name": "tables", + "type": "CONCEPT_LIST", + "label": { + "de": "Konzepte", + "en": "Concepts" + }, + "isTwoDimensional": false, + "conceptDropzoneLabel": { + "en": "Add a concept or a concept list", + "de": "Füge ein Konzept oder eine Konzeptliste hinzu" + }, + "validations": [ + "NOT_EMPTY" + ] + } + ] } From 56924de534db4d9d5b3553eb464e20db123b4f0c Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 7 Jun 2023 17:14:00 +0200 Subject: [PATCH 44/45] use HTTPHealthCheck instead of custom impl doing the same thing --- .../external/form/ExternalFormBackendApi.java | 18 ++++++++---------- .../form/ExternalFormBackendHealthCheck.java | 15 --------------- 2 files changed, 8 insertions(+), 25 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendHealthCheck.java diff --git a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java index d62be767e6..42e3fe35bd 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java +++ b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java @@ -21,6 +21,7 @@ import com.bakdata.conquery.models.datasets.Dataset; import com.codahale.metrics.health.HealthCheck; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.dropwizard.health.check.http.HttpHealthCheck; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -96,7 +97,8 @@ public ExternalTaskState postForm(ExternalForm form, User originalUser, User ser .header(HTTP_HEADER_CQ_AUTHENTICATION, serviceUserToken) .header(HTTP_HEADER_CQ_AUTHENTICATION_ORIGINAL, originalUserToken); - return request.post(Entity.entity(form, MediaType.APPLICATION_JSON_TYPE), ExternalTaskState.class); + ExternalTaskState post = request.post(Entity.entity(form.getExternalApiPayload(), MediaType.APPLICATION_JSON_TYPE), ExternalTaskState.class); + return post; } public ExternalTaskState getFormState(UUID externalId) { @@ -113,14 +115,10 @@ public Response getResult(final URI resultURL) { } - public HealthCheck.Result checkHealth() { - log.trace("Checking health from: {}", getHealthTarget); - try { - getHealthTarget.request(MediaType.APPLICATION_JSON_TYPE).get(Void.class); - return HealthCheck.Result.healthy(); - } - catch (Exception e) { - return HealthCheck.Result.unhealthy(e.getMessage()); - } + public HealthCheck createHealthCheck() { + return new HttpHealthCheck( + getHealthTarget.getUri().toString(), client + ); } + } diff --git a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendHealthCheck.java b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendHealthCheck.java deleted file mode 100644 index c0d87ea4cd..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendHealthCheck.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.bakdata.conquery.io.external.form; - -import com.codahale.metrics.health.HealthCheck; -import lombok.RequiredArgsConstructor; - -@RequiredArgsConstructor -public class ExternalFormBackendHealthCheck extends HealthCheck { - - private final ExternalFormBackendApi externalApi; - - @Override - protected Result check() throws Exception { - return externalApi.checkHealth(); - } -} From 649ea9a39ea4dc16265518ceaa0287f536154242 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 7 Jun 2023 17:14:36 +0200 Subject: [PATCH 45/45] properly format apiPayload for ExternalForm --- .../conquery/apiv1/forms/ExternalForm.java | 13 +- .../models/config/FormBackendConfig.java | 11 +- .../integration/common/IntegrationUtils.java | 117 +++++++++--------- .../tests/ExternalFormBackendTest.java | 44 +++---- 4 files changed, 88 insertions(+), 97 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java index 49ec7b6ca2..cc5fc0cc8b 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java @@ -37,6 +37,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.deser.ContextualDeserializer; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.base.Strings; import lombok.AllArgsConstructor; import lombok.Getter; @@ -60,16 +61,22 @@ public class ExternalForm extends Form implements SubTyped { @JsonValue @ToString.Exclude private final ObjectNode node; + private final String subType; + + public JsonNode getExternalApiPayload() { + return ((ObjectNode) node.deepCopy() + .without("values")) + .set("type", new TextNode(subType)); + + } @Nullable @Override @JsonIgnore public JsonNode getValues() { - return node; + return node.get("values"); } - private final String subType; - @Override public String getLocalizedTypeLabel() { final JsonNode formTitle = node.get("title"); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java index da0e4f9ae9..d592a8f691 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java @@ -17,7 +17,6 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.external.form.ExternalFormBackendApi; -import com.bakdata.conquery.io.external.form.ExternalFormBackendHealthCheck; import com.bakdata.conquery.io.external.form.ExternalFormMixin; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; @@ -94,7 +93,9 @@ public void initialize(ManagerNode managerNode) { client.register(new JacksonMessageBodyProvider(om)); // Register health check - managerNode.getEnvironment().healthChecks().register(getId(), new ExternalFormBackendHealthCheck(createApi())); + final ExternalFormBackendApi externalApi = createApi(); + + managerNode.getEnvironment().healthChecks().register(getId(), externalApi.createHealthCheck()); // Register form configuration provider managerNode.getFormScanner().registerFrontendFormConfigProvider(this::registerFormConfigs); @@ -121,11 +122,11 @@ public boolean supportsFormType(String formType) { * @param formConfigs Collection to add received form configs to. */ private void registerFormConfigs(ImmutableCollection.Builder formConfigs) { - Set supportedFormTypes = new HashSet<>(); + final Set supportedFormTypes = new HashSet<>(); for (ObjectNode formConfig : createApi().getFormConfigs()) { final String subType = formConfig.get("type").asText(); - String formType = createSubTypedId(subType); + final String formType = createSubTypedId(subType); // Override type with our subtype formConfig.set("type", new TextNode(formType)); @@ -155,7 +156,7 @@ public User createServiceUser(User originalUser, Dataset dataset) { // the actual user and download permissions. final User serviceUser = - managerNode.getAuthController().flatCopyUser(originalUser, String.format("%s_%s", this.getClass().getSimpleName().toLowerCase(), getId())); + managerNode.getAuthController().flatCopyUser(originalUser, String.format("%s_%s", getClass().getSimpleName().toLowerCase(), getId())); // The user is able to read the dataset, ensure that the service user can download results serviceUser.addPermission(dataset.createPermission(Ability.DOWNLOAD.asSet())); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java index be2ccc74e5..5f34d01a92 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java @@ -44,10 +44,10 @@ public static void importPermissionConstellation(MetaStorage storage, Role[] rol } for (RequiredUser rUser : rUsers) { - User user = rUser.getUser(); + final User user = rUser.getUser(); storage.addUser(user); - RoleId[] rolesInjected = rUser.getRolesInjected(); + final RoleId[] rolesInjected = rUser.getRolesInjected(); for (RoleId mandatorId : rolesInjected) { user.addRole(storage.getRole(mandatorId)); @@ -60,45 +60,6 @@ public static Query parseQuery(StandaloneSupport support, JsonNode rawQuery) thr return ConqueryTestSpec.parseSubTree(support, rawQuery, Query.class); } - - private static URI getPostQueryURI(StandaloneSupport conquery) { - return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), DatasetQueryResource.class, "postQuery") - .buildFromMap(Map.of( - "dataset", conquery.getDataset().getId() - )); - } - - private static JsonNode getRawExecutionStatus(String id, StandaloneSupport conquery, User user) { - final URI queryStatusURI = getQueryStatusURI(conquery, id); - // We try at most 5 times, queryStatus waits for 10s, we therefore don't need to timeout here. - // Query getQueryStatus until it is no longer running. - for (int trial = 0; trial < 5; trial++) { - log.debug("Trying to get Query result"); - - JsonNode execStatusRaw = - conquery.getClient() - .target(queryStatusURI) - .request(MediaType.APPLICATION_JSON_TYPE) - .header("Authorization", "Bearer " + conquery.getAuthorizationController().getConqueryTokenRealm().createTokenForUser(user.getId())) - .get(JsonNode.class); - - String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText(); - - if (!ExecutionState.RUNNING.name().equals(status)) { - return execStatusRaw; - } - } - - throw new IllegalStateException("Query was running too long."); - } - - private static URI getQueryStatusURI(StandaloneSupport conquery, String id) { - return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "getStatus") - .buildFromMap(Map.of( - "query", id, "dataset", conquery.getDataset().getId() - )); - } - /** * Send a query onto the conquery instance and assert the result's size. * @@ -112,15 +73,15 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O .createTokenForUser(user.getId()); // Submit Query - Response response = conquery.getClient() - .target(postQueryURI) - .request(MediaType.APPLICATION_JSON_TYPE) - .header("Authorization", "Bearer " + userToken) - .post(Entity.entity(query, MediaType.APPLICATION_JSON_TYPE)); + final Response response = conquery.getClient() + .target(postQueryURI) + .request(MediaType.APPLICATION_JSON_TYPE) + .header("Authorization", "Bearer " + userToken) + .post(Entity.entity(query, MediaType.APPLICATION_JSON_TYPE)); assertThat(response.getStatusInfo().getStatusCode()).as("Result of %s", postQueryURI) - .isEqualTo(expectedResponseCode); + .isEqualTo(expectedResponseCode); if (expectedState == ExecutionState.FAILED && !response.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL)) { return null; @@ -132,14 +93,14 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O // TODO implement this properly: ExecutionStatus status = response.readEntity(ExecutionStatus.Full.class); - JsonNode execStatusRaw = getRawExecutionStatus(id, conquery, user); + final JsonNode execStatusRaw = getRawExecutionStatus(id, conquery, user); - String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText(); - long numberOfResults = execStatusRaw.get(ExecutionStatus.Fields.numberOfResults).asLong(0); + final String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText(); + final long numberOfResults = execStatusRaw.get(ExecutionStatus.Fields.numberOfResults).asLong(0); assertThat(status).isEqualTo(expectedState.name()); - if (expectedState == ExecutionState.DONE && expectedSize != -1) { + if (expectedState == ExecutionState.DONE && expectedSize != -1) { assertThat(numberOfResults) .describedAs("Query results") .isEqualTo(expectedSize); @@ -148,22 +109,60 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O return ManagedExecutionId.Parser.INSTANCE.parse(id); } + private static URI getPostQueryURI(StandaloneSupport conquery) { + return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), DatasetQueryResource.class, "postQuery") + .buildFromMap(Map.of( + "dataset", conquery.getDataset().getId() + )); + } + + private static JsonNode getRawExecutionStatus(String id, StandaloneSupport conquery, User user) { + final URI queryStatusURI = getQueryStatusURI(conquery, id); + // We try at most 5 times, queryStatus waits for 10s, we therefore don't need to timeout here. + // Query getQueryStatus until it is no longer running. + for (int trial = 0; trial < 5; trial++) { + log.debug("Trying to get Query result"); + + final JsonNode execStatusRaw = + conquery.getClient() + .target(queryStatusURI) + .request(MediaType.APPLICATION_JSON_TYPE) + .header("Authorization", "Bearer " + conquery.getAuthorizationController().getConqueryTokenRealm().createTokenForUser(user.getId())) + .get(JsonNode.class); + + final String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText(); + + if (!ExecutionState.RUNNING.name().equals(status)) { + return execStatusRaw; + } + } + + throw new IllegalStateException("Query was running too long."); + } + + private static URI getQueryStatusURI(StandaloneSupport conquery, String id) { + return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "getStatus") + .buildFromMap(Map.of( + "query", id, "dataset", conquery.getDataset().getId() + )); + } + public static FullExecutionStatus getExecutionStatus(StandaloneSupport conquery, ManagedExecutionId executionId, User user, int expectedResponseCode) { final URI queryStatusURI = getQueryStatusURI(conquery, executionId.toString()); final String userToken = conquery.getAuthorizationController() - .getConqueryTokenRealm() - .createTokenForUser(user.getId()); + .getConqueryTokenRealm() + .createTokenForUser(user.getId()); - Response response = conquery.getClient() - .target(queryStatusURI) - .request(MediaType.APPLICATION_JSON_TYPE) - .header("Authorization", "Bearer " + userToken) - .get(); + final Response response = conquery.getClient() + .target(queryStatusURI) + .request(MediaType.APPLICATION_JSON_TYPE) + .header("Authorization", "Bearer " + userToken) + .get(); assertThat(response.getStatusInfo().getStatusCode()).as("Result of %s", queryStatusURI) - .isEqualTo(expectedResponseCode); + .isEqualTo(expectedResponseCode); return response.readEntity(FullExecutionStatus.class); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java index 58ed118406..3931d12eeb 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.integration.tests; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; import static org.mockserver.model.HttpRequest.request; import java.io.File; @@ -34,6 +33,7 @@ import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import org.mockserver.integration.ClientAndServer; +import org.mockserver.mock.Expectation; import org.mockserver.mock.OpenAPIExpectation; import org.mockserver.model.HttpResponse; import org.mockserver.model.StringBody; @@ -56,7 +56,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { .getEnvironment() .healthChecks() .runHealthCheck(FORM_BACKEND_ID) - .isHealthy()).describedAs("Checking health of form backend").isTrue(); + .isHealthy()) + .describedAs("Checking health of form backend").isTrue(); log.info("Get external form configs"); final FormScanner formScanner = testConquery.getStandaloneCommand().getManager().getFormScanner(); @@ -76,11 +77,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Generate asset urls and check them in the status - final UriBuilder apiUriBuilder = testConquery.getSupport(name) - .defaultApiURIBuilder(); - final ManagedExecution storedExecution = testConquery.getSupport(name) - .getMetaStorage() - .getExecution(managedExecutionId); + final UriBuilder apiUriBuilder = testConquery.getSupport(name).defaultApiURIBuilder(); + final ManagedExecution storedExecution = testConquery.getSupport(name).getMetaStorage().getExecution(managedExecutionId); final URI downloadURLasset1 = ResultExternalResource.getDownloadURL(apiUriBuilder.clone(), (ManagedExecution & ExternalResult) storedExecution, executionStatus.getResultUrls() @@ -94,18 +92,12 @@ public void execute(String name, TestConquery testConquery) throws Exception { assertThat(executionStatus.getStatus()).isEqualTo(ExecutionState.DONE); - assertThat(executionStatus.getResultUrls()).containsExactly( - new ResultAsset("Result", downloadURLasset1), - new ResultAsset("Another Result", downloadURLasset2) - ); + assertThat(executionStatus.getResultUrls()).containsExactly(new ResultAsset("Result", downloadURLasset1), new ResultAsset("Another Result", downloadURLasset2)); log.info("Download Result"); final String response = - support.getClient() - .target(executionStatus.getResultUrls().get(0).url()) - .request(javax.ws.rs.core.MediaType.TEXT_PLAIN_TYPE) - .get(String.class); + support.getClient().target(executionStatus.getResultUrls().get(0).url()).request(javax.ws.rs.core.MediaType.TEXT_PLAIN_TYPE).get(String.class); assertThat(response).isEqualTo("Hello"); @@ -139,6 +131,7 @@ public ConqueryConfig overrideConfig(ConqueryConfig conf, File workdir) { return conf.withStorage(storageConfig.withDirectory(storageDir)); } + @SneakyThrows @NotNull private URI createFormServer() throws IOException { log.info("Starting mock form backend server"); @@ -146,26 +139,17 @@ private URI createFormServer() throws IOException { final URI baseURI = URI.create(String.format("http://127.0.0.1:%d", formBackend.getPort())); - formBackend.upsert(OpenAPIExpectation.openAPIExpectation("/com/bakdata/conquery/external/openapi-form-backend.yaml")); + Expectation[] expectations = formBackend.upsert(OpenAPIExpectation.openAPIExpectation("/com/bakdata/conquery/external/openapi-form-backend.yaml")); + // Result request matcher - formBackend.when(request("/result.txt")) - .respond(HttpResponse.response() - .withBody(StringBody.exact("Hello") - ) - ); + formBackend.when(request("/result.txt")).respond(HttpResponse.response().withBody(StringBody.exact("Hello"))); + // Trap: Log failed request formBackend.when(request()).respond(httpRequest -> { - log.error( - "{} on {}\n\t Headers: {}\n\tBody {}", - httpRequest.getMethod(), - httpRequest.getPath(), - httpRequest.getHeaderList(), - httpRequest.getBodyAsString() - ); - fail("Trapped because request did not match. See log."); - throw new Error("Received unmappable request"); + log.error("{} on {}\n\t Headers: {}\n\tBody {}", httpRequest.getMethod(), httpRequest.getPath(), httpRequest.getHeaderList(), httpRequest.getBodyAsString()); + return HttpResponse.notFoundResponse(); }); return baseURI; }