Merge pull request #2989 from ingef/release

Merge Release

.github/workflows/codeql-analysis.yml

@@ -48,7 +48,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -59,7 +59,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl

.github/workflows/lint_openapi.yml

@@ -3,6 +3,7 @@ on:
   pull_request:
     paths:
       - "openapi.yaml"
+      - "backend/src/main/resources/com/bakdata/conquery/external/openapi-form-backend.yaml"
       - ".github/workflows/lint_openapi.yaml"
 jobs:
   lint:
@@ -11,5 +12,7 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: "Lint OpenApi spec"
+      - name: "Lint OpenApi spec for /api"
         run: vacuum lint --details ./openapi.yaml
+      - name: "Lint OpenApi spec for external form backends"
+        run: vacuum lint --details ./backend/src/main/resources/com/bakdata/conquery/external/openapi-form-backend.yaml

README.md

@@ -72,6 +72,7 @@ To run the end-to-end test locally:
 For further informations on this and other tests, please refer to the corresponding [CI configuration](https://github.com/bakdata/conquery/tree/develop/.github/workflows).
 
 ### Data Integration
+
 To make you own data (in form of CSVs) availiable to conquery some steps are necessary:
 
 1. Describe your data table structure with meta data, by generating Import- and Table-JSONs.
@@ -81,10 +82,16 @@ To make you own data (in form of CSVs) availiable to conquery some steps are nec
 
 To get a better impression of the single steps, take a look at the [Tutorials](./tutorial/mimic_iii_demo/README.md)
 
+### Custom Forms
+
+It is possible to extend the analytic capabilities by adding custom forms.
+Custom forms provide an easy way for a user to perform standardized analyses.
+For more informations, take a look at the [guide](./docs/custom_forms.md)
 
 ## Acknowledgements
 
-This platform was created by [InGef – Institut für angewandte Gesundheitsforschung Berlin GmbH](http://www.ingef.de/) in cooperation with [bakdata GmbH](http://www.bakdata.com) and [Kai Rollmann](https://kairollmann.de/).
+This platform was created by [InGef – Institut für angewandte Gesundheitsforschung Berlin GmbH](http://www.ingef.de/) in
+cooperation with [bakdata GmbH](http://www.bakdata.com) and [Kai Rollmann](https://kairollmann.de/).
 
 [<img alt="InGef – Institut für angewandte Gesundheitsforschung Berlin GmbH" src="images/ingef_logo.svg" height=50 align="top">](http://www.ingef.de/)
 &emsp;

autodoc/src/main/java/com/bakdata/conquery/Constants.java

@@ -16,13 +16,13 @@
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
 
-import com.bakdata.conquery.apiv1.ExecutionStatus;
 import com.bakdata.conquery.apiv1.FilterTemplate;
-import com.bakdata.conquery.apiv1.FullExecutionStatus;
 import com.bakdata.conquery.apiv1.IdLabel;
 import com.bakdata.conquery.apiv1.KeyValue;
 import com.bakdata.conquery.apiv1.MetaDataPatch;
-import com.bakdata.conquery.apiv1.OverviewExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.ExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.FullExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus;
 import com.bakdata.conquery.apiv1.frontend.FrontendRoot;
 import com.bakdata.conquery.apiv1.frontend.FrontendValue;
 import com.bakdata.conquery.apiv1.query.CQElement;

backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java

@@ -1,6 +1,6 @@
 package com.bakdata.conquery.apiv1;
 
-import java.net.URL;
+import java.net.URI;
 import java.util.List;
 
 import javax.validation.constraints.NotEmpty;
@@ -10,7 +10,7 @@
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.io.jackson.serializer.NsIdRef;
 import com.bakdata.conquery.io.storage.NamespaceStorage;
-import com.bakdata.conquery.models.config.SearchConfig;
+import com.bakdata.conquery.models.config.IndexConfig;
 import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.datasets.concepts.Searchable;
 import com.bakdata.conquery.models.identifiable.IdentifiableImpl;
@@ -19,6 +19,7 @@
 import com.bakdata.conquery.models.index.FrontendValueIndexKey;
 import com.bakdata.conquery.models.index.IndexService;
 import com.bakdata.conquery.models.index.search.SearchIndex;
+import com.bakdata.conquery.util.io.FileUtil;
 import com.bakdata.conquery.util.search.TrieSearch;
 import com.fasterxml.jackson.annotation.JacksonInject;
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -40,7 +41,7 @@
 @ToString
 @Slf4j
 @CPSType(id = "CSV_TEMPLATE", base = SearchIndex.class)
-public class FilterTemplate extends IdentifiableImpl<SearchIndexId> implements Searchable, SearchIndex {
+public class FilterTemplate extends IdentifiableImpl<SearchIndexId> implements Searchable<SearchIndexId>, SearchIndex {
 
 	private static final long serialVersionUID = 1L;
 
@@ -55,7 +56,7 @@ public class FilterTemplate extends IdentifiableImpl<SearchIndexId> implements S
 	 * Path to CSV File.
 	 */
 	@NotNull
-	private final URL filePath;
+	private final URI filePath;
 
 	/**
 	 * Value to be sent for filtering.
@@ -90,15 +91,18 @@ public boolean isSearchDisabled() {
 		return false;
 	}
 
-	public List<TrieSearch<FrontendValue>> getSearches(SearchConfig config, NamespaceStorage storage) {
+	public List<TrieSearch<FrontendValue>> getSearches(IndexConfig config, NamespaceStorage storage) {
+
+		final URI resolvedURI = FileUtil.getResolvedUri(config.getBaseUrl(), getFilePath());
+		log.trace("Resolved filter template reference url for search '{}': {}", this.getId(), resolvedURI);
 
 		FrontendValueIndex search = indexService.getIndex(new FrontendValueIndexKey(
-				filePath,
+				resolvedURI,
 				columnValue,
 				value,
 				optionValue,
 				isGenerateSuffixes() ? getMinSuffixLength() : Integer.MAX_VALUE,
-				config.getSplit()
+				config.getSearchSplitChars()
 		));
 
 		return List.of(search);

backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java

@@ -1,12 +1,14 @@
 package com.bakdata.conquery.apiv1;
 
-import java.net.URL;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
 import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
@@ -21,6 +23,10 @@
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriBuilder;
 
+import com.bakdata.conquery.apiv1.execution.ExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.FullExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus;
+import com.bakdata.conquery.apiv1.execution.ResultAsset;
 import com.bakdata.conquery.apiv1.query.CQElement;
 import com.bakdata.conquery.apiv1.query.ConceptQuery;
 import com.bakdata.conquery.apiv1.query.ExternalUpload;
@@ -46,6 +52,7 @@
 import com.bakdata.conquery.models.config.ColumnConfig;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.datasets.Dataset;
+import com.bakdata.conquery.models.datasets.PreviewConfig;
 import com.bakdata.conquery.models.datasets.SecondaryIdDescription;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.error.ConqueryError;
@@ -96,7 +103,7 @@ public ManagedExecution postQuery(Dataset dataset, QueryDescription query, Subje
 		// This maps works as long as we have query visitors that are not configured in anyway.
 		// So adding a visitor twice would replace the previous one but both would have yielded the same result.
 		// For the future a better data structure might be desired that also regards similar QueryVisitors of different configuration
-		ClassToInstanceMap<QueryVisitor> visitors = MutableClassToInstanceMap.create();
+		final ClassToInstanceMap<QueryVisitor> visitors = MutableClassToInstanceMap.create();
 		query.addVisitors(visitors);
 
 		// Initialize checks that need to traverse the query tree
@@ -184,7 +191,7 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId exe
 			execution = newExecution;
 		}
 
-		ExecutionState state = execution.getState();
+		final ExecutionState state = execution.getState();
 		if (state.equals(ExecutionState.RUNNING)) {
 			log.trace("The Execution[{}] was already started and its state is: {}", execution.getId(), state);
 			return execution;
@@ -200,7 +207,7 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId exe
 
 
 	public Stream<ExecutionStatus> getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) {
-		Collection<ManagedExecution> allQueries = storage.getAllExecutions();
+		final Collection<ManagedExecution> allQueries = storage.getAllExecutions();
 
 		return getQueriesFiltered(dataset, RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders);
 	}
@@ -217,9 +224,9 @@ public Stream<ExecutionStatus> getQueriesFiltered(Dataset datasetId, UriBuilder
 						 .filter(q -> q.getState().equals(ExecutionState.DONE) || q.getState().equals(ExecutionState.NEW))
 						 .filter(q -> subject.isPermitted(q, Ability.READ))
 						 .map(mq -> {
-							 OverviewExecutionStatus status = mq.buildStatusOverview(uriBuilder.clone(), subject);
+							 final OverviewExecutionStatus status = mq.buildStatusOverview(uriBuilder.clone(), subject);
 							 if (mq.isReadyToDownload()) {
-								 status.setResultUrls(getDownloadUrls(config.getResultProviders(), mq, uriBuilder, allProviders));
+								 status.setResultUrls(getResultAssets(config.getResultProviders(), mq, uriBuilder, allProviders));
 							 }
 							 return status;
 						 });
@@ -236,12 +243,21 @@ public Stream<ExecutionStatus> getQueriesFiltered(Dataset datasetId, UriBuilder
 	 * @param allProviders If true, forces {@link ResultRendererProvider} to return an URL if possible.
 	 * @return The modified status
 	 */
-	public static List<URL> getDownloadUrls(List<ResultRendererProvider> renderer, ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) {
+	public static List<ResultAsset> getResultAssets(List<ResultRendererProvider> renderer, ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) {
 
 		return renderer.stream()
-					   .map(r -> r.generateResultURLs(exec, uriBuilder.clone(), allProviders))
+					   .map(r -> {
+						   try {
+							   return r.generateResultURLs(exec, uriBuilder.clone(), allProviders);
+						   }
+						   catch (MalformedURLException | URISyntaxException e) {
+							   log.error("Cannot generate result urls for execution '{}' with provider '{}'", exec.getId(), r.getClass().getName());
+							   return null;
+						   }
+					   })
+					   .filter(Objects::nonNull)
 					   .flatMap(Collection::stream)
-					   .collect(Collectors.toList());
+					   .toList();
 
 	}
 
@@ -305,11 +321,11 @@ public void patchQuery(Subject subject, ManagedExecution execution, MetaDataPatc
 
 		// TODO remove this, since we don't translate anymore
 		// Patch this query in other datasets
-		List<Dataset> remainingDatasets = datasetRegistry.getAllDatasets();
+		final List<Dataset> remainingDatasets = datasetRegistry.getAllDatasets();
 		remainingDatasets.remove(execution.getDataset());
 
 		for (Dataset dataset : remainingDatasets) {
-			ManagedExecutionId id = new ManagedExecutionId(dataset.getId(), execution.getQueryId());
+			final ManagedExecutionId id = new ManagedExecutionId(dataset.getId(), execution.getQueryId());
 			final ManagedExecution otherExecution = storage.getExecution(id);
 			if (otherExecution == null) {
 				continue;
@@ -349,7 +365,7 @@ public FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject su
 		final FullExecutionStatus status = query.buildStatusFull(subject);
 
 		if (query.isReadyToDownload() && subject.isPermitted(query.getDataset(), Ability.DOWNLOAD)) {
-			status.setResultUrls(getDownloadUrls(config.getResultProviders(), query, url, allProviders));
+			status.setResultUrls(getResultAssets(config.getResultProviders(), query, url, allProviders));
 		}
 		return status;
 	}
@@ -402,8 +418,8 @@ public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, Ext
 	public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uriBuilder, String idKind, String entity, List<Connector> sources, Dataset dataset, Range<LocalDate> dateRange) {
 
 		final Namespace namespace = datasetRegistry.get(dataset.getId());
-		EntityPreviewForm form =
-				EntityPreviewForm.create(entity, idKind, dateRange, sources, namespace.getPreviewConfig().getSelects());
+		final PreviewConfig previewConfig = datasetRegistry.get(dataset.getId()).getPreviewConfig();
+		final EntityPreviewForm form = EntityPreviewForm.create(entity, idKind, dateRange, sources, previewConfig.getSelects(), previewConfig.getTimeStratifiedSelects(), datasetRegistry);
 
 		// TODO make sure that subqueries are also system
 		// TODO do not persist system queries
@@ -421,8 +437,8 @@ public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uri
 		}
 
 
-		FullExecutionStatus status = execution.buildStatusFull(subject);
-		status.setResultUrls(getDownloadUrls(config.getResultProviders(), execution, uriBuilder, false));
+		final FullExecutionStatus status = execution.buildStatusFull(subject);
+		status.setResultUrls(getResultAssets(config.getResultProviders(), execution, uriBuilder, false));
 		return status;
 	}
 

backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java → backend/src/main/java/com/bakdata/conquery/apiv1/execution/ExecutionStatus.java

@@ -1,4 +1,9 @@
-package com.bakdata.conquery.apiv1;
+package com.bakdata.conquery.apiv1.execution;
+
+import java.time.LocalDateTime;
+import java.time.ZonedDateTime;
+import java.util.Collections;
+import java.util.List;
 
 import com.bakdata.conquery.models.execution.ExecutionState;
 import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
@@ -10,12 +15,6 @@
 import lombok.ToString;
 import lombok.experimental.FieldNameConstants;
 
-import java.net.URL;
-import java.time.LocalDateTime;
-import java.time.ZonedDateTime;
-import java.util.Collections;
-import java.util.List;
-
 @NoArgsConstructor
 @ToString
 @Data
@@ -49,7 +48,7 @@ public abstract class ExecutionStatus {
 	/**
 	 * The urls under from which the result of the execution can be downloaded as soon as it finished successfully.
 	 */
-	private List<URL> resultUrls = Collections.emptyList();
+	private List<ResultAsset> resultUrls = Collections.emptyList();
 
 
 }

backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java → backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java

@@ -1,4 +1,10 @@
-package com.bakdata.conquery.apiv1;
+package com.bakdata.conquery.apiv1.execution;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import javax.annotation.Nullable;
 
 import com.bakdata.conquery.apiv1.query.QueryDescription;
 import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection;
@@ -11,11 +17,6 @@
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldNameConstants;
 
-import javax.annotation.Nullable;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
-
 /**
  * This status holds extensive information about the query description and meta data that is computational heavy
  * and can produce a larger payload to requests.

backend/src/main/java/com/bakdata/conquery/apiv1/OverviewExecutionStatus.java → backend/src/main/java/com/bakdata/conquery/apiv1/execution/OverviewExecutionStatus.java

@@ -1,4 +1,4 @@
-package com.bakdata.conquery.apiv1;
+package com.bakdata.conquery.apiv1.execution;
 
 import lombok.NoArgsConstructor;
 

backend/src/main/java/com/bakdata/conquery/apiv1/execution/ResultAsset.java

@@ -0,0 +1,31 @@
+package com.bakdata.conquery.apiv1.execution;
+
+import java.net.URI;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import io.dropwizard.validation.ValidationMethod;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+@Slf4j
+public record ResultAsset(String label, URI url) {
+
+	@JsonIgnore
+	@ValidationMethod(message = "Generated assetId was blank")
+	public boolean isAssetIdValid() {
+		return StringUtils.isBlank(getAssetId());
+	}
+
+	/**
+	 * An url-encoded id of this asset.
+	 * It derives from the last path section of the file part in this url.
+	 */
+	@JsonIgnore
+	public String getAssetId() {
+
+		final String path = url.getPath();
+		return URLEncoder.encode(path.substring(path.lastIndexOf('/') + 1), StandardCharsets.UTF_8);
+	}
+}