Merge pull request #3118 from ingef/release

Merge Release

backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java

@@ -86,7 +86,7 @@ public String getLocalizedTypeLabel() {
 
 		// Form had no specific title set. Try localized lookup in FormConfig
 		final Locale preferredLocale = I18n.LOCALE.get();
-		final FormType frontendConfig = FormScanner.FRONTEND_FORM_CONFIGS.get(getFormType());
+		final FormType frontendConfig = FormScanner.resolveFormType(getFormType());
 
 		if (frontendConfig == null) {
 			return getSubType();

backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java

@@ -8,7 +8,9 @@
 import com.bakdata.conquery.models.auth.entities.Subject;
 import com.bakdata.conquery.models.auth.permissions.Ability;
 import com.bakdata.conquery.models.datasets.Dataset;
+import com.bakdata.conquery.models.error.ConqueryError;
 import com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner;
+import com.bakdata.conquery.models.forms.frontendconfiguration.FormType;
 import com.bakdata.conquery.models.forms.managed.ManagedForm;
 import com.bakdata.conquery.models.query.visitor.QueryVisitor;
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -42,7 +44,13 @@ public String getFormType() {
 	public void authorize(Subject subject, Dataset submittedDataset, @NonNull ClassToInstanceMap<QueryVisitor> visitors, MetaStorage storage) {
 		QueryDescription.super.authorize(subject, submittedDataset, visitors, storage);
 		// Check if subject is allowed to create this form
-		subject.authorize(FormScanner.FRONTEND_FORM_CONFIGS.get(getFormType()), Ability.CREATE);
+		final FormType formType = FormScanner.resolveFormType(getFormType());
+
+		if (formType == null) {
+			throw new ConqueryError.ExecutionCreationErrorUnspecified();
+		}
+
+		subject.authorize(formType, Ability.CREATE);
 	}
 
 

backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java

@@ -1,6 +1,7 @@
 package com.bakdata.conquery.apiv1.frontend;
 
 import java.net.URL;
+import java.time.LocalDate;
 
 import com.bakdata.conquery.models.config.FrontendConfig;
 import com.bakdata.conquery.models.config.IdColumnConfig;
@@ -19,6 +20,7 @@ public record FrontendConfiguration(
 		FrontendConfig.CurrencyConfig currency,
 		IdColumnConfig queryUpload,
 		URL manualUrl,
-		String contactEmail
+		String contactEmail,
+		LocalDate observationPeriodStart
 ) {
 }

backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendPreviewConfig.java

@@ -1,6 +1,5 @@
 package com.bakdata.conquery.apiv1.frontend;
 
-import java.time.LocalDate;
 import java.util.Collection;
 import java.util.List;
 
@@ -19,8 +18,6 @@ public static class Labelled {
 		private final String label;
 	}
 
-	private final LocalDate observationPeriodMin;
-
 	private final Collection<Labelled> all;
 	@JsonProperty("default")
 	private final Collection<Labelled> defaultConnectors;

backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java

@@ -2,6 +2,7 @@
 
 import java.time.LocalDate;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -143,17 +144,26 @@ public void resolve(QueryResolveContext context) {
 		query.resolve(context);
 
 		// First is dates, second is source id
-		AtomicInteger currentPosition = new AtomicInteger(2);
+		final AtomicInteger currentPosition = new AtomicInteger(2);
 
 		final Map<SecondaryIdDescription, Integer> secondaryIdPositions = calculateSecondaryIdPositions(currentPosition);
 
-		positions = calculateColumnPositions(currentPosition, tables, secondaryIdPositions);
+		// We need to know if a column is a concept column so we can prioritize it if it is also a SecondaryId
+		final Set<Column> conceptColumns = tables.stream()
+												 .map(CQConcept::getTables)
+												 .flatMap(Collection::stream)
+												 .map(CQTable::getConnector)
+												 .map(Connector::getColumn)
+												 .filter(Objects::nonNull)
+												 .collect(Collectors.toSet());
 
-		resultInfos = createResultInfos(secondaryIdPositions);
+		positions = calculateColumnPositions(currentPosition, tables, secondaryIdPositions, conceptColumns);
+
+		resultInfos = createResultInfos(secondaryIdPositions, conceptColumns);
 	}
 
 	private Map<SecondaryIdDescription, Integer> calculateSecondaryIdPositions(AtomicInteger currentPosition) {
-		Map<SecondaryIdDescription, Integer> secondaryIdPositions = new HashMap<>();
+		final Map<SecondaryIdDescription, Integer> secondaryIdPositions = new HashMap<>();
 
 		// SecondaryIds are pulled to the front and grouped over all tables
 		tables.stream()
@@ -169,9 +179,10 @@ private Map<SecondaryIdDescription, Integer> calculateSecondaryIdPositions(Atomi
 		return secondaryIdPositions;
 	}
 
-	private static Map<Column, Integer> calculateColumnPositions(AtomicInteger currentPosition, List<CQConcept> tables, Map<SecondaryIdDescription, Integer> secondaryIdPositions) {
+	private static Map<Column, Integer> calculateColumnPositions(AtomicInteger currentPosition, List<CQConcept> tables, Map<SecondaryIdDescription, Integer> secondaryIdPositions, Set<Column> conceptColumns) {
 		final Map<Column, Integer> positions = new HashMap<>();
 
+
 		for (CQConcept concept : tables) {
 			for (CQTable table : concept.getTables()) {
 
@@ -188,7 +199,8 @@ private static Map<Column, Integer> calculateColumnPositions(AtomicInteger curre
 						continue;
 					}
 
-					if (column.getSecondaryId() != null) {
+					// We want to have ConceptColumns separate here.
+					if (column.getSecondaryId() != null && !conceptColumns.contains(column)) {
 						positions.putIfAbsent(column, secondaryIdPositions.get(column.getSecondaryId()));
 						continue;
 					}
@@ -201,7 +213,7 @@ private static Map<Column, Integer> calculateColumnPositions(AtomicInteger curre
 		return positions;
 	}
 
-	private List<ResultInfo> createResultInfos(Map<SecondaryIdDescription, Integer> secondaryIdPositions) {
+	private List<ResultInfo> createResultInfos(Map<SecondaryIdDescription, Integer> secondaryIdPositions, Set<Column> conceptColumns) {
 
 		final int size = positions.values().stream().mapToInt(i -> i).max().getAsInt() + 1;
 
@@ -252,7 +264,7 @@ private List<ResultInfo> createResultInfos(Map<SecondaryIdDescription, Integer>
 			}
 
 			// SecondaryIds and date columns are pulled to the front, thus already covered.
-			if (column.getSecondaryId() != null) {
+			if (column.getSecondaryId() != null && !conceptColumns.contains(column)) {
 				infos[secondaryIdPositions.get(column.getSecondaryId())].getSemantics()
 																		.add(new SemanticType.ColumnT(column));
 				continue;
@@ -303,12 +315,12 @@ public static String printValue(Concept concept, Object rawValue, PrintSettings
 
 		final TreeConcept tree = (TreeConcept) concept;
 
-		int localId = (int) rawValue;
+		final int localId = (int) rawValue;
 
 		final ConceptTreeNode<?> node = tree.getElementByLocalId(localId);
 
 		if (!printSettings.isPrettyPrint()) {
-			return node.getId().toStringWithoutDataset();
+			return node.getId().toString();
 		}
 
 		if (node.getDescription() == null) {

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java

@@ -19,6 +19,7 @@
 import com.bakdata.conquery.models.identifiable.ids.specific.FilterId;
 import com.bakdata.conquery.models.query.QueryResolveContext;
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -27,6 +28,7 @@
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
 import com.fasterxml.jackson.databind.exc.InvalidTypeIdException;
+import io.dropwizard.validation.ValidationMethod;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
@@ -44,6 +46,10 @@
 @EqualsAndHashCode
 @ToString(of = "value")
 public abstract class FilterValue<VALUE> {
+	/**
+	 * Very large SELECT FilterValues can cause issues, so we just limit it to large but not gigantic quantities.
+	 */
+	private static final int MAX_NUMBER_FILTER_VALUES = 20_000;
 	@NotNull
 	@Nonnull
 	@NsIdRef
@@ -68,6 +74,12 @@ public static class CQMultiSelectFilter extends FilterValue<String[]> {
 		public CQMultiSelectFilter(@NsIdRef Filter<String[]> filter, String[] value) {
 			super(filter, value);
 		}
+
+		@ValidationMethod(message = "Too many values selected.")
+		@JsonIgnore
+		public boolean isSaneAmountOfFilterValues() {
+			return getValue().length < MAX_NUMBER_FILTER_VALUES;
+		}
 	}
 
 	@NoArgsConstructor
@@ -77,6 +89,12 @@ public static class CQBigMultiSelectFilter extends FilterValue<String[]> {
 		public CQBigMultiSelectFilter(@NsIdRef Filter<String[]> filter, String[] value) {
 			super(filter, value);
 		}
+
+		@ValidationMethod(message = "Too many values selected.")
+		@JsonIgnore
+		public boolean isSaneAmountOfFilterValues() {
+			return getValue().length < MAX_NUMBER_FILTER_VALUES;
+		}
 	}
 
 	@NoArgsConstructor

backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java

@@ -46,6 +46,7 @@
 import com.bakdata.conquery.resources.unprotected.AuthServlet;
 import com.bakdata.conquery.tasks.PermissionCleanupTask;
 import com.bakdata.conquery.tasks.QueryCleanupTask;
+import com.bakdata.conquery.tasks.ReloadMetaStorageTask;
 import com.bakdata.conquery.tasks.ReportConsistencyTask;
 import com.bakdata.conquery.util.io.ConqueryMDC;
 import com.fasterxml.jackson.databind.DeserializationConfig;
@@ -194,8 +195,9 @@ public void run(ConqueryConfig config, Environment environment) throws Interrupt
 				)));
 		environment.admin().addTask(new PermissionCleanupTask(storage));
 		environment.admin().addTask(new ReportConsistencyTask(datasetRegistry));
+		environment.admin().addTask(new ReloadMetaStorageTask(storage));
 
-		ShutdownTask shutdown = new ShutdownTask();
+		final ShutdownTask shutdown = new ShutdownTask();
 		environment.admin().addTask(shutdown);
 		environment.lifecycle().addServerLifecycleListener(shutdown);
 	}

backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java

@@ -314,23 +314,27 @@ private void reportJobManagerStatus() {
 
 
 		// Collect the ShardNode and all its workers jobs into a single queue
-		final JobManagerStatus jobManagerStatus = jobManager.reportStatus();
 
 		for (Worker worker : workers.getWorkers().values()) {
-			jobManagerStatus.getJobs().addAll(worker.getJobManager().reportStatus().getJobs());
-		}
-
+			final JobManagerStatus jobManagerStatus = new JobManagerStatus(
+					null, worker.getInfo().getDataset(),
+					worker.getJobManager().getJobStatus()
+			);
 
-		try {
-			context.trySend(new UpdateJobManagerStatus(jobManagerStatus));
-		}
-		catch (Exception e) {
-			log.warn("Failed to report job manager status", e);
+			try {
+				context.trySend(new UpdateJobManagerStatus(jobManagerStatus));
+			}
+			catch (Exception e) {
+				log.warn("Failed to report job manager status", e);
 
-			if (config.isFailOnError()) {
-				System.exit(1);
+				if (config.isFailOnError()) {
+					System.exit(1);
+				}
 			}
 		}
+
+
+
 	}
 
 	public boolean isBusy() {

backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java

@@ -12,7 +12,7 @@
 public class NoSuchElementExceptionMapper implements ExceptionMapper<NoSuchElementException> {
 	@Override
 	public Response toResponse(NoSuchElementException exception) {
-		log.trace("Mapping exception:", exception);
+		log.warn("Uncaught NoSuchElementException", exception);
 		return Response.status(Response.Status.NOT_FOUND).type(MediaType.APPLICATION_JSON_TYPE).entity(exception.getMessage()).build();
 	}
 }

backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java

@@ -6,31 +6,33 @@
 import javax.annotation.Nullable;
 import javax.validation.Valid;
 import javax.validation.constraints.Email;
+import javax.validation.constraints.Min;
 import javax.validation.constraints.NotNull;
 
 import com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner;
 import com.fasterxml.jackson.annotation.JsonAlias;
-import groovy.transform.ToString;
 import lombok.AllArgsConstructor;
 import lombok.Data;
-import lombok.Getter;
 import lombok.NoArgsConstructor;
-import lombok.Setter;
 import lombok.With;
 import lombok.extern.slf4j.Slf4j;
 
-@ToString
-@Getter
-@Setter
 @NoArgsConstructor
 @AllArgsConstructor
 @Slf4j
 @With
+@Data
 public class FrontendConfig {
 	@Valid
 	@NotNull
 	private CurrencyConfig currency = new CurrencyConfig();
 
+	/**
+	 * Years to include in entity preview.
+	 */
+	@Min(0)
+	private int observationPeriodYears = 6;
+
 	/**
 	 * The url that points a manual. This is also used by the {@link FormScanner}
 	 * as the base url for forms that specify a relative url. Internally {@link URI#resolve(URI)}

backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java

@@ -1,6 +1,6 @@
 package com.bakdata.conquery.models.datasets;
 
-import java.time.LocalDate;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
@@ -46,11 +46,6 @@
 @NoArgsConstructor
 public class PreviewConfig {
 
-	/**
-	 * Default start-date for EntityPreview, end date will always be LocalDate.now()
-	 */
-	@NotNull
-	private LocalDate observationStart;
 
 	/**
 	 * Selects to be used in {@link com.bakdata.conquery.apiv1.QueryProcessor#getSingleEntityExport(Subject, UriBuilder, String, String, List, Dataset, Range)}.
@@ -120,17 +115,18 @@ public record InfoCardSelect(@NotNull String label, SelectId select, String desc
 	 * Defines a group of selects that will be evaluated per quarter and year in the requested period of the entity-preview.
 	 */
 	public record TimeStratifiedSelects(@NotNull String label, String description, @NotEmpty List<InfoCardSelect> selects){
-		@ValidationMethod(message = "Selects may be referenced only once.")
-		@JsonIgnore
-		public boolean isSelectsUnique() {
-			return selects().stream().map(InfoCardSelect::select).distinct().count() == selects().size();
-		}
+	}
 
-		@ValidationMethod(message = "Labels must be unique.")
-		@JsonIgnore
-		public boolean isLabelsUnique() {
-			return selects().stream().map(InfoCardSelect::label).distinct().count() == selects().size();
-		}
+	@ValidationMethod(message = "Selects may be referenced only once.")
+	@JsonIgnore
+	public boolean isSelectsUnique() {
+		return timeStratifiedSelects.stream().map(TimeStratifiedSelects::selects).flatMap(Collection::stream).map(InfoCardSelect::select).distinct().count() == timeStratifiedSelects.stream().map(TimeStratifiedSelects::selects).flatMap(Collection::stream).count();
+	}
+
+	@ValidationMethod(message = "Labels must be unique.")
+	@JsonIgnore
+	public boolean isLabelsUnique() {
+		return timeStratifiedSelects.stream().map(TimeStratifiedSelects::selects).flatMap(Collection::stream).map(InfoCardSelect::label).distinct().count() == timeStratifiedSelects.stream().map(TimeStratifiedSelects::selects).flatMap(Collection::stream).count();
 	}
 
 	@JsonIgnore

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java

@@ -185,12 +185,11 @@ public static FrontendTable createTable(Connector con) {
 														  .collect(Collectors.toSet()))
 							 .build();
 
-		if (con.getValidityDates().size() > 1) {
-			result.setDateColumn(new FrontendValidityDate(con.getValidityDatesDescription(), null, con.getValidityDates()
-																									  .stream()
-																									  .map(vd -> new FrontendValue(vd.getId()
-																																	 .toString(), vd.getLabel()))
-																									  .collect(Collectors.toList())));
+		if (!con.getValidityDates().isEmpty()) {
+			result.setDateColumn(new FrontendValidityDate(con.getValidityDatesDescription(), null,
+														  con.getValidityDates().stream()
+															 .map(vd -> new FrontendValue(vd.getId().toString(), vd.getLabel()))
+															 .collect(Collectors.toList())));
 
 			if (!result.getDateColumn().getOptions().isEmpty()) {
 				result.getDateColumn().setDefaultValue(result.getDateColumn().getOptions().get(0).getValue());