From e1388096d20d7c89814accc60648a122e04d7949 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 30 Jan 2023 18:01:53 +0100 Subject: [PATCH 001/149] capsule result urls in api object --- .../java/com/bakdata/conquery/Constants.java | 6 +++--- .../conquery/apiv1/QueryProcessor.java | 21 +++++++++++-------- .../{ => execution}/ExecutionStatus.java | 15 +++++++------ .../{ => execution}/FullExecutionStatus.java | 13 ++++++------ .../OverviewExecutionStatus.java | 2 +- .../conquery/apiv1/execution/ResultAsset.java | 6 ++++++ .../apiv1/query/TableExportQuery.java | 2 +- .../ResultRender/ResultRendererProvider.java | 4 ++-- .../models/config/ArrowResultProvider.java | 8 +++---- .../models/config/CsvResultProvider.java | 6 +++--- .../models/config/ExcelResultProvider.java | 5 +++-- .../models/config/ParquetResultProvider.java | 8 +++---- .../models/execution/ManagedExecution.java | 6 +++--- .../models/forms/managed/ManagedForm.java | 2 +- .../conquery/models/query/ManagedQuery.java | 4 ++-- .../query/preview/EntityPreviewExecution.java | 2 +- .../query/preview/EntityPreviewStatus.java | 2 +- .../resources/admin/rest/AdminResource.java | 2 +- .../resources/api/DatasetQueryResource.java | 4 ++-- .../conquery/resources/api/QueryResource.java | 2 +- .../api/StoredQueriesProcessorTest.java | 12 ++++------- .../integration/DownloadLinkGeneration.java | 6 ++++-- .../integration/common/IntegrationUtils.java | 4 ++-- .../integration/tests/EntityExportTest.java | 2 ++ .../integration/tests/ReusedQueryTest.java | 3 +-- 25 files changed, 77 insertions(+), 70 deletions(-) rename backend/src/main/java/com/bakdata/conquery/apiv1/{ => execution}/ExecutionStatus.java (92%) rename backend/src/main/java/com/bakdata/conquery/apiv1/{ => execution}/FullExecutionStatus.java (97%) rename backend/src/main/java/com/bakdata/conquery/apiv1/{ => execution}/OverviewExecutionStatus.java (83%) create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/execution/ResultAsset.java diff --git a/autodoc/src/main/java/com/bakdata/conquery/Constants.java b/autodoc/src/main/java/com/bakdata/conquery/Constants.java index 36c3fe1fe4..005ad4a0b4 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/Constants.java +++ b/autodoc/src/main/java/com/bakdata/conquery/Constants.java @@ -16,13 +16,13 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; -import com.bakdata.conquery.apiv1.ExecutionStatus; import com.bakdata.conquery.apiv1.FilterTemplate; -import com.bakdata.conquery.apiv1.FullExecutionStatus; import com.bakdata.conquery.apiv1.IdLabel; import com.bakdata.conquery.apiv1.KeyValue; import com.bakdata.conquery.apiv1.MetaDataPatch; -import com.bakdata.conquery.apiv1.OverviewExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus; import com.bakdata.conquery.apiv1.frontend.FrontendRoot; import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.apiv1.query.CQElement; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index e6394d3543..fa226a35d8 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -2,7 +2,6 @@ import static com.bakdata.conquery.models.auth.AuthorizationHelper.buildDatasetAbilityMap; -import java.net.URL; import java.time.LocalDate; import java.util.Collection; import java.util.List; @@ -21,6 +20,10 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.UriBuilder; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.ExternalUpload; @@ -191,7 +194,7 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId } - public Stream getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) { + public Stream getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) { Collection> allQueries = storage.getAllExecutions(); return getQueriesFiltered(dataset, RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders); @@ -212,7 +215,7 @@ public Stream getQueriesFiltered(Dataset datasetId, UriBuilder .map(mq -> { OverviewExecutionStatus status = mq.buildStatusOverview(uriBuilder.clone(), subject); if (mq.isReadyToDownload(datasetAbilities)) { - status.setResultUrls(getDownloadUrls(config.getResultProviders(), mq, uriBuilder, allProviders)); + status.setResultUrls(getResultAssets(config.getResultProviders(), mq, uriBuilder, allProviders)); } return status; }); @@ -229,7 +232,7 @@ public Stream getQueriesFiltered(Dataset datasetId, UriBuilder * @param allProviders If true, forces {@link ResultRendererProvider} to return an URL if possible. * @return The modified status */ - public static List getDownloadUrls(List renderer, ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { + public static List getResultAssets(List renderer, ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { return renderer.stream() .map(r -> r.generateResultURLs(exec, uriBuilder.clone(), allProviders)) @@ -332,15 +335,15 @@ public void deleteQuery(Subject subject, ManagedExecution execution) { storage.removeExecution(execution.getId()); } - public FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject subject, UriBuilder url, Boolean allProviders) { + public com.bakdata.conquery.apiv1.execution.FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject subject, UriBuilder url, Boolean allProviders) { query.initExecutable(datasetRegistry, config); Map> datasetAbilities = buildDatasetAbilityMap(subject, datasetRegistry); - final FullExecutionStatus status = query.buildStatusFull(storage, subject, datasetRegistry, config); + final com.bakdata.conquery.apiv1.execution.FullExecutionStatus status = query.buildStatusFull(storage, subject, datasetRegistry, config); if (query.isReadyToDownload(datasetAbilities)) { - status.setResultUrls(getDownloadUrls(config.getResultProviders(), query, url, allProviders)); + status.setResultUrls(getResultAssets(config.getResultProviders(), query, url, allProviders)); } return status; } @@ -389,7 +392,7 @@ public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, Ext /** * Create and submit {@link EntityPreviewForm} for subject on to extract sources for entity, and extract some additional infos to be used as infocard. */ - public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uriBuilder, String idKind, String entity, List sources, Dataset dataset, Range dateRange) { + public com.bakdata.conquery.apiv1.execution.FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uriBuilder, String idKind, String entity, List sources, Dataset dataset, Range dateRange) { EntityPreviewForm form = EntityPreviewForm.create(entity, idKind, dateRange, sources, datasetRegistry.get(dataset.getId()).getPreviewConfig().getSelects()); @@ -411,7 +414,7 @@ public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uri FullExecutionStatus status = execution.buildStatusFull(storage, subject, datasetRegistry, config); - status.setResultUrls(getDownloadUrls(config.getResultProviders(), execution, uriBuilder, false)); + status.setResultUrls(getResultAssets(config.getResultProviders(), execution, uriBuilder, false)); return status; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ExecutionStatus.java similarity index 92% rename from backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java rename to backend/src/main/java/com/bakdata/conquery/apiv1/execution/ExecutionStatus.java index a9c674b11e..4aec080f22 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ExecutionStatus.java @@ -1,4 +1,9 @@ -package com.bakdata.conquery.apiv1; +package com.bakdata.conquery.apiv1.execution; + +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.Collections; +import java.util.List; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; @@ -10,12 +15,6 @@ import lombok.ToString; import lombok.experimental.FieldNameConstants; -import java.net.URL; -import java.time.LocalDateTime; -import java.time.ZonedDateTime; -import java.util.Collections; -import java.util.List; - @NoArgsConstructor @ToString @Data @@ -49,7 +48,7 @@ public abstract class ExecutionStatus { /** * The urls under from which the result of the execution can be downloaded as soon as it finished successfully. */ - private List resultUrls = Collections.emptyList(); + private List resultUrls = Collections.emptyList(); } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java similarity index 97% rename from backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java rename to backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java index 61c5d4a917..17c058884d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java @@ -1,4 +1,10 @@ -package com.bakdata.conquery.apiv1; +package com.bakdata.conquery.apiv1.execution; + +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection; @@ -11,11 +17,6 @@ import lombok.NoArgsConstructor; import lombok.experimental.FieldNameConstants; -import javax.annotation.Nullable; -import java.util.Collection; -import java.util.List; -import java.util.Set; - /** * This status holds extensive information about the query description and meta data that is computational heavy * and can produce a larger payload to requests. diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/OverviewExecutionStatus.java b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/OverviewExecutionStatus.java similarity index 83% rename from backend/src/main/java/com/bakdata/conquery/apiv1/OverviewExecutionStatus.java rename to backend/src/main/java/com/bakdata/conquery/apiv1/execution/OverviewExecutionStatus.java index 3e06d0e74c..f68c83ec96 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/OverviewExecutionStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/OverviewExecutionStatus.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.apiv1; +package com.bakdata.conquery.apiv1.execution; import lombok.NoArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ResultAsset.java b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ResultAsset.java new file mode 100644 index 0000000000..1457bc1dc6 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ResultAsset.java @@ -0,0 +1,6 @@ +package com.bakdata.conquery.apiv1.execution; + +import java.net.URL; + +public record ResultAsset(String label, URL url) { +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index aa190fe549..6d85e72edf 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -19,7 +19,7 @@ import javax.validation.constraints.NotNull; import com.bakdata.conquery.ConqueryConstants; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.io.cps.CPSType; diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java b/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java index df1714e6f5..2dc64b25e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.io.result.ResultRender; -import java.net.URL; import java.util.Collection; import javax.ws.rs.core.UriBuilder; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.models.execution.ManagedExecution; @@ -24,7 +24,7 @@ public interface ResultRendererProvider { * @param allProviders A flag that should override internal "hide-this-url" flags. * @return An Optional with the url or an empty optional. */ - Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders); + Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders); void registerResultResource(DropwizardResourceConfig environment, ManagerNode manager); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ArrowResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ArrowResultProvider.java index 2e828c6edb..40317e9847 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ArrowResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ArrowResultProvider.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.config; import java.net.MalformedURLException; -import java.net.URL; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -10,6 +9,7 @@ import javax.validation.constraints.NotNull; import javax.ws.rs.core.UriBuilder; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; @@ -33,7 +33,7 @@ public class ArrowResultProvider implements ResultRendererProvider { @Override @SneakyThrows(MalformedURLException.class) - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { + public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { if (!(exec instanceof SingleTableResult)) { return Collections.emptyList(); } @@ -43,8 +43,8 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuilder u } return List.of( - ResultArrowResource.getFileDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec), - ResultArrowResource.getStreamDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec) + new ResultAsset("Arrow File", ResultArrowResource.getFileDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec)), + new ResultAsset("Arrow Stream", ResultArrowResource.getStreamDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec)) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultProvider.java index ee2edfa3ea..3c7a4ddc3d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultProvider.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.models.config; import java.net.MalformedURLException; -import java.net.URL; import java.util.Collection; import java.util.Collections; import java.util.List; import javax.ws.rs.core.UriBuilder; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; @@ -30,7 +30,7 @@ public class CsvResultProvider implements ResultRendererProvider { private boolean hidden = false; @SneakyThrows(MalformedURLException.class) - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { + public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { if (!(exec instanceof SingleTableResult)) { return Collections.emptyList(); } @@ -39,7 +39,7 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuilder u return Collections.emptyList(); } - return List.of(ResultCsvResource.getDownloadURL(uriBuilder, (ManagedExecution & SingleTableResult) exec)); + return List.of(new ResultAsset("CSV", ResultCsvResource.getDownloadURL(uriBuilder, (ManagedExecution & SingleTableResult) exec))); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java index 139d08196a..706097198d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java @@ -11,6 +11,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.UriBuilder; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; @@ -45,7 +46,7 @@ public class ExcelResultProvider implements ResultRendererProvider { @Override @SneakyThrows(MalformedURLException.class) - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { + public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { // We only support/produce xlsx files with one sheet for now if (!(exec instanceof SingleTableResult singleExecution)) { log.trace("Execution result is not a single table"); @@ -82,7 +83,7 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuilder u final URL resultUrl = ResultExcelResource.getDownloadURL(uriBuilder, (ManagedExecution & SingleTableResult) exec); log.trace("Generated URL: {}", resultUrl); - return List.of(resultUrl); + return List.of(new ResultAsset("XLSX", resultUrl)); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ParquetResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ParquetResultProvider.java index 68bfb1c210..c32d728bdf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ParquetResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ParquetResultProvider.java @@ -1,21 +1,19 @@ package com.bakdata.conquery.models.config; import java.net.MalformedURLException; -import java.net.URL; import java.util.Collection; import java.util.Collections; import java.util.List; import javax.ws.rs.core.UriBuilder; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; -import com.bakdata.conquery.io.result.arrow.ResultArrowProcessor; import com.bakdata.conquery.io.result.parquet.ResultParquetProcessor; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.resources.api.ResultArrowResource; import com.bakdata.conquery.resources.api.ResultParquetResource; import io.dropwizard.jersey.DropwizardResourceConfig; import lombok.Data; @@ -30,7 +28,7 @@ public class ParquetResultProvider implements ResultRendererProvider { @Override @SneakyThrows(MalformedURLException.class) - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { + public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { if (!(exec instanceof SingleTableResult)) { return Collections.emptyList(); } @@ -40,7 +38,7 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuilder u } return List.of( - ResultParquetResource.getDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec) + new ResultAsset("PARQUET", ResultParquetResource.getDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec)) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java index 71e097357c..bc1c9385ad 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java @@ -17,9 +17,9 @@ import javax.validation.constraints.NotNull; import javax.ws.rs.core.UriBuilder; -import com.bakdata.conquery.apiv1.ExecutionStatus; -import com.bakdata.conquery.apiv1.FullExecutionStatus; -import com.bakdata.conquery.apiv1.OverviewExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.jackson.serializer.MetaIdRef; diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java index 2ac8379861..dc9dece1db 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java @@ -6,7 +6,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.forms.Form; import com.bakdata.conquery.apiv1.forms.FormConfigAPI; import com.bakdata.conquery.apiv1.query.QueryDescription; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index 376fc37996..df8ff022d3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -13,8 +13,8 @@ import java.util.stream.Stream; import c10n.C10N; -import com.bakdata.conquery.apiv1.ExecutionStatus; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index b5fdfc280f..72ef84d85c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -6,7 +6,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.Subject; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewStatus.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewStatus.java index 64b4de088c..26a664b6b9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewStatus.java @@ -3,7 +3,7 @@ import java.util.List; import java.util.Set; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.models.query.ColumnDescriptor; import com.bakdata.conquery.models.types.SemanticType; import lombok.Data; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java index eec99abc41..1049cd2757 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java @@ -23,7 +23,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.UriBuilder; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.io.jersey.ExtraMimeTypes; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.Subject; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetQueryResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetQueryResource.java index 7997c6ac5f..1d84b57ed2 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetQueryResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetQueryResource.java @@ -22,10 +22,10 @@ import javax.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; -import com.bakdata.conquery.apiv1.ExecutionStatus; -import com.bakdata.conquery.apiv1.FullExecutionStatus; import com.bakdata.conquery.apiv1.QueryProcessor; import com.bakdata.conquery.apiv1.RequestAwareUriBuilder; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.ExternalUpload; import com.bakdata.conquery.apiv1.query.ExternalUploadResult; import com.bakdata.conquery.apiv1.query.QueryDescription; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java index 210cd63543..be0756a98f 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java @@ -19,10 +19,10 @@ import javax.ws.rs.core.Context; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; -import com.bakdata.conquery.apiv1.FullExecutionStatus; import com.bakdata.conquery.apiv1.MetaDataPatch; import com.bakdata.conquery.apiv1.QueryProcessor; import com.bakdata.conquery.apiv1.RequestAwareUriBuilder; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.execution.ManagedExecution; diff --git a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java index 0b123abfcd..78097cf627 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java @@ -3,7 +3,6 @@ import static com.bakdata.conquery.models.execution.ExecutionState.*; import static org.assertj.core.api.Assertions.assertThat; -import java.net.URL; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; @@ -14,9 +13,10 @@ import javax.ws.rs.core.UriBuilder; -import com.bakdata.conquery.apiv1.ExecutionStatus; -import com.bakdata.conquery.apiv1.OverviewExecutionStatus; import com.bakdata.conquery.apiv1.QueryProcessor; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.apiv1.query.ConceptQuery; @@ -48,10 +48,6 @@ import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.resources.api.ResultArrowResource; -import com.bakdata.conquery.resources.api.ResultCsvResource; -import com.bakdata.conquery.resources.api.ResultExcelResource; -import com.bakdata.conquery.resources.api.ResultParquetResource; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.google.common.collect.ImmutableList; import lombok.SneakyThrows; @@ -233,7 +229,7 @@ public List getResultInfos() { status.setNumberOfResults(resultCount); status.setSecondaryId(secondaryId); // This is probably not interesting on the overview (only if there is an filter for the search) if(state.equals(DONE)) { - List resultUrls = new ArrayList<>(); + List resultUrls = new ArrayList<>(); resultUrls.addAll(EXCEL_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); resultUrls.addAll(CSV_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); resultUrls.addAll(ARROW_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java index cd4395227e..7f8e28b585 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java @@ -5,7 +5,8 @@ import java.net.URL; import java.util.Set; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.integration.common.IntegrationUtils; import com.bakdata.conquery.integration.json.JsonIntegrationTest; import com.bakdata.conquery.integration.json.QueryTest; @@ -67,7 +68,8 @@ public void execute(StandaloneSupport conquery) throws Exception { FullExecutionStatus status = IntegrationUtils.getExecutionStatus(conquery, exec.getId(), user, 200); // This Url is missing the `/api` path part, because we use the standard UriBuilder here - assertThat(status.getResultUrls()).contains(new URL(String.format("%s/result/%s.csv", conquery.defaultApiURIBuilder().toString(), exec.getId()))); + assertThat(status.getResultUrls()).contains(new ResultAsset("CSV", new URL(String.format("%s/result/%s.csv", conquery.defaultApiURIBuilder() + .toString(), exec.getId())))); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java index 571b9ccdd8..f765d107bc 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java @@ -10,8 +10,8 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import com.bakdata.conquery.apiv1.ExecutionStatus; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.integration.json.ConqueryTestSpec; import com.bakdata.conquery.io.storage.MetaStorage; diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java index 53847259bc..c52a2264c6 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java @@ -19,6 +19,7 @@ import javax.ws.rs.core.Response; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.integration.common.LoadingUtil; import com.bakdata.conquery.integration.common.RequiredData; import com.bakdata.conquery.integration.json.JsonIntegrationTest; @@ -174,6 +175,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final Optional csvUrl = result.getResultUrls().stream() + .map(ResultAsset::url) .filter(url -> url.getFile().endsWith(".csv")) .findFirst(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java index b02e64ab2d..70d34e9dc2 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java @@ -12,7 +12,7 @@ import javax.ws.rs.client.Entity; import javax.ws.rs.core.MediaType; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; @@ -22,7 +22,6 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; import com.bakdata.conquery.integration.common.IntegrationUtils; import com.bakdata.conquery.integration.common.LoadingUtil; -import com.bakdata.conquery.integration.common.RequiredData; import com.bakdata.conquery.integration.json.JsonIntegrationTest; import com.bakdata.conquery.integration.json.QueryTest; import com.bakdata.conquery.io.storage.MetaStorage; From 5061e825a304fb0768b8efd3218862055d97917e Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 31 Jan 2023 18:57:38 +0100 Subject: [PATCH 002/149] adds result provider to path so that an external provider would not collide with providers of a similar name --- .../com/bakdata/conquery/resources/api/ResultArrowResource.java | 2 +- .../com/bakdata/conquery/resources/api/ResultCsvResource.java | 2 +- .../com/bakdata/conquery/resources/api/ResultExcelResource.java | 2 +- .../bakdata/conquery/resources/api/ResultParquetResource.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowResource.java index 310cce1e5e..f5338f28af 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowResource.java @@ -29,7 +29,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -@Path("result/") +@Path("result/arrow") @RequiredArgsConstructor(onConstructor_ = {@Inject}) public class ResultArrowResource { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java index 86e645be86..30d7b5fd2e 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java @@ -30,7 +30,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -@Path("result/") +@Path("result/csv") @RequiredArgsConstructor(onConstructor_ = {@Inject}) public class ResultCsvResource { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java index ab087cd662..801545b7ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java @@ -29,7 +29,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -@Path("result/") +@Path("result/xlsx") @RequiredArgsConstructor(onConstructor_ = {@Inject}) public class ResultExcelResource { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultParquetResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultParquetResource.java index b2a7610957..822c7c5320 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultParquetResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultParquetResource.java @@ -30,7 +30,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -@Path("result/") +@Path("result/parquet") @RequiredArgsConstructor(onConstructor_ = {@Inject}) public class ResultParquetResource { From ac48e18901e3020210cc186e6becd3dfa40d3a46 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 31 Jan 2023 21:47:22 +0100 Subject: [PATCH 003/149] fix tests --- .../conquery/io/result/ResultUtil.java | 3 +- .../io/result/arrow/ResultArrowProcessor.java | 4 +- .../io/result/csv/ResultCsvProcessor.java | 4 +- .../io/result/excel/ResultExcelProcessor.java | 4 +- .../parquet/ResultParquetProcessor.java | 4 +- .../integration/DownloadLinkGeneration.java | 4 +- .../tests/endpoints/apiEndpointInfo.json | 48 +++++++++---------- 7 files changed, 36 insertions(+), 35 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java index 47c80e2ba9..cff45a3070 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java @@ -77,7 +77,8 @@ public static void checkSingleTableResult(ManagedExecution exec) { } - public static void authorizeExecutable(Subject subject, ManagedExecution exec, Dataset dataset) { + public static void authorizeExecutable(Subject subject, ManagedExecution exec) { + final Dataset dataset = exec.getDataset(); subject.authorize(dataset, Ability.READ); subject.authorize(dataset, Ability.DOWNLOAD); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index 61a32303d0..26fe56bca2 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -99,9 +99,9 @@ public static & SingleTableResult> Response getAr final Dataset dataset = exec.getDataset(); - log.info("Downloading results for {} on dataset {}", exec, dataset); + log.info("Downloading results for {}", exec.getId()); - ResultUtil.authorizeExecutable(subject, exec, dataset); + ResultUtil.authorizeExecutable(subject, exec); if (!(exec instanceof ManagedQuery || (exec instanceof ManagedForm && ((ManagedForm) exec).getSubQueries().size() == 1))) { return Response.status(HttpStatus.SC_UNPROCESSABLE_ENTITY, "Execution result is not a single Table").build(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index 0c6b7c2375..ab2ccaa790 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -45,9 +45,9 @@ public & SingleTableResult> Response createResult final Namespace namespace = datasetRegistry.get(dataset.getId()); ConqueryMDC.setLocation(subject.getName()); - log.info("Downloading results for {} on dataset {}", exec, dataset); + log.info("Downloading results for {}", exec.getId()); - ResultUtil.authorizeExecutable(subject, exec, dataset); + ResultUtil.authorizeExecutable(subject, exec); // Check if subject is permitted to download on all datasets that were referenced by the query authorizeDownloadDatasets(subject, exec); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index f96e906683..f6b34821d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -44,9 +44,9 @@ public & SingleTableResult> Response createResult final Dataset dataset = exec.getDataset(); - log.info("Downloading results for {} on dataset {}", exec, dataset); + log.info("Downloading results for {}", exec.getId()); - ResultUtil.authorizeExecutable(subject, exec, dataset); + ResultUtil.authorizeExecutable(subject, exec); ResultUtil.checkSingleTableResult(exec); subject.authorize(dataset, Ability.DOWNLOAD); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java index bee9a72e10..7a9e25a067 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java @@ -41,9 +41,9 @@ public Response createResultFile(Subject subject, ManagedExecution exec, bool final Dataset dataset = exec.getDataset(); - log.info("Downloading results for {} on dataset {}", exec, dataset); + log.info("Downloading results for {}", exec.getId()); - ResultUtil.authorizeExecutable(subject, exec, dataset); + ResultUtil.authorizeExecutable(subject, exec); ResultUtil.checkSingleTableResult(exec); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java index 7f8e28b585..51a6080b77 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java @@ -68,8 +68,8 @@ public void execute(StandaloneSupport conquery) throws Exception { FullExecutionStatus status = IntegrationUtils.getExecutionStatus(conquery, exec.getId(), user, 200); // This Url is missing the `/api` path part, because we use the standard UriBuilder here - assertThat(status.getResultUrls()).contains(new ResultAsset("CSV", new URL(String.format("%s/result/%s.csv", conquery.defaultApiURIBuilder() - .toString(), exec.getId())))); + assertThat(status.getResultUrls()).contains(new ResultAsset("CSV", new URL(String.format("%s/result/csv/%s.csv", conquery.defaultApiURIBuilder() + .toString(), exec.getId())))); } } diff --git a/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json b/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json index 18984d7fe3..7439ccb6eb 100644 --- a/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json +++ b/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json @@ -45,30 +45,30 @@ "clazz": "QueryResource" }, { - "method": "GET", - "path": "/result/{query}.csv", - "clazz": "ResultCsvResource" - }, - { - "method": "GET", - "path": "/result/{query}.xlsx", - "clazz": "ResultExcelResource" - }, - { - "method": "GET", - "path": "/result/{query}.arrs", - "clazz": "ResultArrowResource" - }, - { - "method": "GET", - "path": "/result/{query}.arrf", - "clazz": "ResultArrowResource" - }, - { - "method": "GET", - "path": "/result/{query}.parquet", - "clazz": "ResultParquetResource" - }, + "method": "GET", + "path": "/result/csv/{query}.csv", + "clazz": "ResultCsvResource" + }, + { + "method": "GET", + "path": "/result/xlsx/{query}.xlsx", + "clazz": "ResultExcelResource" + }, + { + "method": "GET", + "path": "/result/arrow/{query}.arrs", + "clazz": "ResultArrowResource" + }, + { + "method": "GET", + "path": "/result/arrow/{query}.arrf", + "clazz": "ResultArrowResource" + }, + { + "method": "GET", + "path": "/result/parquet/{query}.parquet", + "clazz": "ResultParquetResource" + }, { "method": "GET", "path": "/datasets/{dataset}/queries", From 185f05b8d95ccf3bd7112695c2cbce4ecd79780f Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 1 Feb 2023 15:26:20 +0100 Subject: [PATCH 004/149] generalize result name normalization --- .../java/com/bakdata/conquery/io/result/ResultUtil.java | 8 ++++---- .../conquery/io/result/arrow/ResultArrowProcessor.java | 2 +- .../conquery/io/result/csv/ResultCsvProcessor.java | 8 +++++++- .../conquery/io/result/excel/ResultExcelProcessor.java | 3 ++- .../io/result/parquet/ResultParquetProcessor.java | 4 ++-- .../com/bakdata/conquery/resources/ResourceConstants.java | 7 ++++++- .../resources/admin/rest/AuthOverviewResource.java | 2 +- .../bakdata/conquery/resources/api/ResultCsvResource.java | 5 +++-- .../main/java/com/bakdata/conquery/util/io/FileUtil.java | 4 ++-- .../com/bakdata/conquery/io/result/ResultNameTest.java | 4 ++-- 10 files changed, 30 insertions(+), 17 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java index cff45a3070..e8e0a7b421 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java @@ -35,12 +35,12 @@ String getHeaderValue() { } - public static Response makeResponseWithFileName(Response.ResponseBuilder response, String label, String fileExtension, MediaType mediaType, ContentDispositionOption disposition) { + public static Response makeResponseWithFileName(Response.ResponseBuilder response, String filename, MediaType mediaType, ContentDispositionOption disposition) { response.header(HttpHeaders.CONTENT_TYPE, mediaType); - if (!(Strings.isNullOrEmpty(label) || label.isBlank())) { - // Set filename from label if the label was set, otherwise the browser will name the file according to the request path + if (!(Strings.isNullOrEmpty(filename) || filename.isBlank())) { + // Set filename from filename if the filename was set, otherwise the browser will name the file according to the request path response.header("Content-Disposition", String.format( - "%s; filename=\"%s\"", disposition.getHeaderValue(), FileUtil.makeSafeFileName(label, fileExtension))); + "%s; filename=\"%s\"", disposition.getHeaderValue(), FileUtil.makeSafeFileName(filename))); } return response.build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index 26fe56bca2..baf6757aca 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -142,7 +142,7 @@ public static & SingleTableResult> Response getAr } }; - return makeResponseWithFileName(Response.ok(out), exec.getLabelWithoutAutoLabelSuffix(), fileExtension, mediaType, ResultUtil.ContentDispositionOption.ATTACHMENT); + return makeResponseWithFileName(Response.ok(out), String.join(".", exec.getLabelWithoutAutoLabelSuffix(), fileExtension), mediaType, ResultUtil.ContentDispositionOption.ATTACHMENT); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index ab2ccaa790..7fbcd6a40f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -25,6 +25,7 @@ import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.resources.ResourceConstants; import com.bakdata.conquery.util.io.ConqueryMDC; import com.bakdata.conquery.util.io.IdColumnUtil; import lombok.RequiredArgsConstructor; @@ -74,7 +75,12 @@ public & SingleTableResult> Response createResult } }; - return makeResponseWithFileName(Response.ok(out), exec.getLabelWithoutAutoLabelSuffix(), "csv", new MediaType("text", "csv", charset.toString()), ResultUtil.ContentDispositionOption.ATTACHMENT); + return makeResponseWithFileName( + Response.ok(out), + String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_CSV), + new MediaType("text", "csv", charset.toString()), + ResultUtil.ContentDispositionOption.ATTACHMENT + ); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index f6b34821d6..52987fc54a 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -22,6 +22,7 @@ import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.resources.ResourceConstants; import com.bakdata.conquery.util.io.ConqueryMDC; import com.bakdata.conquery.util.io.IdColumnUtil; import lombok.RequiredArgsConstructor; @@ -63,7 +64,7 @@ public & SingleTableResult> Response createResult log.trace("FINISHED downloading {}", exec.getId()); }; - return makeResponseWithFileName(Response.ok(out), exec.getLabelWithoutAutoLabelSuffix(), "xlsx", MEDIA_TYPE, ResultUtil.ContentDispositionOption.ATTACHMENT); + return makeResponseWithFileName(Response.ok(out), String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_XLSX), MEDIA_TYPE, ResultUtil.ContentDispositionOption.ATTACHMENT); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java index 7a9e25a067..28f17efeb4 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.io.result.parquet; import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName; -import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_PARQUET; import java.util.Locale; @@ -21,6 +20,7 @@ import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.resources.ResourceConstants; import com.bakdata.conquery.resources.api.ResultParquetResource; import com.bakdata.conquery.util.io.ConqueryMDC; import com.bakdata.conquery.util.io.IdColumnUtil; @@ -74,6 +74,6 @@ public Response createResultFile(Subject subject, ManagedExecution exec, bool }; - return makeResponseWithFileName(Response.ok(out), exec.getLabelWithoutAutoLabelSuffix(), FILE_EXTENTION_PARQUET, PARQUET_MEDIA_TYPE, ResultUtil.ContentDispositionOption.ATTACHMENT); + return makeResponseWithFileName(Response.ok(out), String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_PARQUET), PARQUET_MEDIA_TYPE, ResultUtil.ContentDispositionOption.ATTACHMENT); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/ResourceConstants.java b/backend/src/main/java/com/bakdata/conquery/resources/ResourceConstants.java index 27b99c6d60..267d22e181 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/ResourceConstants.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/ResourceConstants.java @@ -35,7 +35,12 @@ public class ResourceConstants { public static final String FILE_EXTENTION_ARROW_FILE = "arrf"; public static final String FILE_EXTENTION_ARROW_STREAM = "arrs"; public static final String FILE_EXTENTION_PARQUET = "parquet"; - + + public static final String FILE_EXTENTION_CSV = "csv"; + + public static final String FILE_EXTENTION_XLSX = "xlsx"; + + /** * Method to generate a data-model of this class's static members so that they * are accessible from within a freemarker template. diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AuthOverviewResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AuthOverviewResource.java index 0e57feb2ae..6d94aac4c8 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AuthOverviewResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AuthOverviewResource.java @@ -39,7 +39,7 @@ public Response getPermissionOverviewAsCSV() { public Response getPermissionOverviewAsCSV(@PathParam(GROUP_ID) Group group) { return Response .ok(processor.getPermissionOverviewAsCSV(group)) - .header("Content-Disposition", String.format("attachment; filename=\"authOverview_%s.csv\"", FileUtil.makeSafeFileName(group.getName(), "csv"))) + .header("Content-Disposition", String.format("attachment; filename=\"authOverview_%s.csv\"", FileUtil.makeSafeFileName(group.getName()))) .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java index 30d7b5fd2e..5a652c5485 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java @@ -9,6 +9,7 @@ import java.util.Optional; import javax.inject.Inject; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.HeaderParam; import javax.ws.rs.Path; @@ -54,11 +55,11 @@ public & SingleTableResult> Response getAsCsv( @PathParam(QUERY) ManagedExecution execution, @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, @QueryParam("charset") String queryCharset, - @QueryParam("pretty") Optional pretty) { + @QueryParam("pretty") @DefaultValue("true") boolean pretty) { checkSingleTableResult(execution); log.info("Result for {} download on dataset {} by subject {} ({}).", execution, execution.getDataset().getId(), subject.getId(), subject.getName()); - return processor.createResult(subject, (E) execution, pretty.orElse(Boolean.TRUE), determineCharset(userAgent, queryCharset)); + return processor.createResult(subject, (E) execution, pretty, determineCharset(userAgent, queryCharset)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/FileUtil.java b/backend/src/main/java/com/bakdata/conquery/util/io/FileUtil.java index 940e31fae5..2a97bbb271 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/FileUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/FileUtil.java @@ -26,8 +26,8 @@ public class FileUtil { public static final Pattern SAVE_FILENAME_REPLACEMENT_MATCHER = Pattern.compile("[^a-zA-Z0-9äÄöÖüÜß .\\-]"); - public static String makeSafeFileName(String label, String fileExtension) { - return SAVE_FILENAME_REPLACEMENT_MATCHER.matcher(label + "." + fileExtension).replaceAll("_"); + public static String makeSafeFileName(String label) { + return SAVE_FILENAME_REPLACEMENT_MATCHER.matcher(label).replaceAll("_"); } public void deleteRecursive(Path path) throws IOException { diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java index abc2aed9cc..dae43745ed 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java @@ -12,7 +12,7 @@ public class ResultNameTest { @Test public void resultNameOk(){ final String label = "azAZ19 äü-ÄÜ"; - String fileName = FileUtil.makeSafeFileName(label, FILE_EXTENSION); + String fileName = FileUtil.makeSafeFileName(label); assertThat(fileName).isEqualTo(label + "." +FILE_EXTENSION); } @@ -20,7 +20,7 @@ public void resultNameOk(){ public void resultNameModified(){ final String label = "()§ $ \\ \" "; - String fileName = FileUtil.makeSafeFileName(label, FILE_EXTENSION); + String fileName = FileUtil.makeSafeFileName(label); assertThat(fileName).isEqualTo("___ _ _ _ ." + FILE_EXTENSION); } } From 7f6a9ba5fe377a9ac4756b4f8706d216c19a4336 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 1 Feb 2023 15:42:03 +0100 Subject: [PATCH 005/149] adds NoSuchElementException->404 mapper --- .../bakdata/conquery/io/jersey/RESTServer.java | 2 ++ .../io/jetty/NoSuchElementExceptionMapper.java | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java diff --git a/backend/src/main/java/com/bakdata/conquery/io/jersey/RESTServer.java b/backend/src/main/java/com/bakdata/conquery/io/jersey/RESTServer.java index b81085af3e..535d33c116 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jersey/RESTServer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jersey/RESTServer.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.io.jetty.ConqueryErrorExecptionMapper; import com.bakdata.conquery.io.jetty.ConqueryJsonExceptionMapper; import com.bakdata.conquery.io.jetty.JsonValidationExceptionMapper; +import com.bakdata.conquery.io.jetty.NoSuchElementExceptionMapper; import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.web.AuthenticationExceptionMapper; import com.bakdata.conquery.models.auth.web.AuthorizationExceptionMapper; @@ -32,6 +33,7 @@ public static void configure(ConqueryConfig config, ResourceConfig jersey) { jersey.register(new AuthenticationExceptionMapper()); jersey.register(new AuthorizationExceptionMapper()); jersey.register(JsonValidationExceptionMapper.class); + jersey.register(NoSuchElementExceptionMapper.class); // default Dropwizard's exception mappers jersey.register(new ConqueryErrorExecptionMapper()); jersey.register(ConqueryJsonExceptionMapper.class); diff --git a/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java b/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java new file mode 100644 index 0000000000..531786b0c5 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java @@ -0,0 +1,18 @@ +package com.bakdata.conquery.io.jetty; + +import java.util.NoSuchElementException; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class NoSuchElementExceptionMapper implements ExceptionMapper { + @Override + public Response toResponse(NoSuchElementException exception) { + log.trace("Mapping exception:", exception); + return Response.status(Response.Status.NOT_FOUND).type(MediaType.APPLICATION_JSON_TYPE).entity(exception.getMessage()).build(); + } +} From cbd8b21fd98e078641774a2ac980a67fcda7f8b1 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 1 Feb 2023 16:05:36 +0100 Subject: [PATCH 006/149] fix ResultNameTest --- .../bakdata/conquery/io/result/ResultNameTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java index dae43745ed..04131c1288 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java @@ -1,26 +1,26 @@ package com.bakdata.conquery.io.result; +import static org.assertj.core.api.Assertions.assertThat; + import com.bakdata.conquery.util.io.FileUtil; import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; - public class ResultNameTest { public static final String FILE_EXTENSION = "test"; @Test - public void resultNameOk(){ + public void resultNameOk() { final String label = "azAZ19 äü-ÄÜ"; - String fileName = FileUtil.makeSafeFileName(label); - assertThat(fileName).isEqualTo(label + "." +FILE_EXTENSION); + String fileName = FileUtil.makeSafeFileName(label + "." + FILE_EXTENSION); + assertThat(fileName).isEqualTo(label + "." + FILE_EXTENSION); } @Test - public void resultNameModified(){ + public void resultNameModified() { final String label = "()§ $ \\ \" "; - String fileName = FileUtil.makeSafeFileName(label); + String fileName = FileUtil.makeSafeFileName(label + "." + FILE_EXTENSION); assertThat(fileName).isEqualTo("___ _ _ _ ." + FILE_EXTENSION); } } From 01cd6cc6ccef60aa1751fb33678c641fd03ac083 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 1 Feb 2023 16:35:44 +0100 Subject: [PATCH 007/149] adapt openapi.yaml --- .../bakdata/conquery/apiv1/QueryProcessor.java | 8 ++++---- openapi.yaml | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index fa226a35d8..8bbd6a88a6 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -194,7 +194,7 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId } - public Stream getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) { + public Stream getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) { Collection> allQueries = storage.getAllExecutions(); return getQueriesFiltered(dataset, RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders); @@ -335,12 +335,12 @@ public void deleteQuery(Subject subject, ManagedExecution execution) { storage.removeExecution(execution.getId()); } - public com.bakdata.conquery.apiv1.execution.FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject subject, UriBuilder url, Boolean allProviders) { + public FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject subject, UriBuilder url, Boolean allProviders) { query.initExecutable(datasetRegistry, config); Map> datasetAbilities = buildDatasetAbilityMap(subject, datasetRegistry); - final com.bakdata.conquery.apiv1.execution.FullExecutionStatus status = query.buildStatusFull(storage, subject, datasetRegistry, config); + final FullExecutionStatus status = query.buildStatusFull(storage, subject, datasetRegistry, config); if (query.isReadyToDownload(datasetAbilities)) { status.setResultUrls(getResultAssets(config.getResultProviders(), query, url, allProviders)); @@ -392,7 +392,7 @@ public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, Ext /** * Create and submit {@link EntityPreviewForm} for subject on to extract sources for entity, and extract some additional infos to be used as infocard. */ - public com.bakdata.conquery.apiv1.execution.FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uriBuilder, String idKind, String entity, List sources, Dataset dataset, Range dateRange) { + public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uriBuilder, String idKind, String entity, List sources, Dataset dataset, Range dateRange) { EntityPreviewForm form = EntityPreviewForm.create(entity, idKind, dateRange, sources, datasetRegistry.get(dataset.getId()).getPreviewConfig().getSelects()); diff --git a/openapi.yaml b/openapi.yaml index 59b480048b..2d8d505421 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -295,9 +295,19 @@ components: description: Available result urls of the query. type: array items: - type: string #TODO url? - format: url - + $ref: "#/components/schemas/result-asset" + result-asset: + type: object + properties: + label: + type: string + minLength: 1 + url: + type: string + format: url + required: + - label + - url labeled: type: object properties: From 1afd0346d5c5886e72a4e3559b127519c706a427 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Wed, 8 Feb 2023 12:14:46 +0100 Subject: [PATCH 008/149] switch to labels, switch internal type, mock api --- frontend/mock-api/index.js | 22 +++++++++++-- frontend/mock-api/stored-queries/25.json | 15 ++++++++- frontend/src/js/api/types.ts | 9 +++-- frontend/src/js/button/DownloadButton.tsx | 33 ++++++++++++++++--- frontend/src/js/button/PreviewButton.tsx | 6 ++-- .../js/button/QueryResultHistoryButton.tsx | 6 ++-- frontend/src/js/entity-history/History.tsx | 4 +-- frontend/src/js/entity-history/actions.ts | 11 ++++--- frontend/src/js/entity-history/reducer.ts | 11 ++++--- frontend/src/js/preview/actions.ts | 8 ++--- frontend/src/js/preview/reducer.ts | 4 +-- .../src/js/previous-queries/list/reducer.ts | 4 +-- .../DownloadResultsDropdownButton.tsx | 19 ++++++----- frontend/src/js/query-runner/QueryResults.tsx | 6 ++-- frontend/src/js/query-runner/reducer.ts | 3 +- 15 files changed, 111 insertions(+), 50 deletions(-) diff --git a/frontend/mock-api/index.js b/frontend/mock-api/index.js index afb4aeb211..63193c2fbf 100644 --- a/frontend/mock-api/index.js +++ b/frontend/mock-api/index.js @@ -151,8 +151,14 @@ module.exports = function (app, port) { status: "DONE", numberOfResults: 5, resultUrls: [ - `/api/results/results.xlsx`, - `/api/results/results.csv`, + { + label: "XLSX", + url: "http://localhost:8080/api/result/xlsx/mimic-iii-demo.51cd95fd-90b2-4573-aab5-11846126427b.xlsx", + }, + { + label: "CSV", + url: "http://localhost:8080/api/result/csv/mimic-iii-demo.51cd95fd-90b2-4573-aab5-11846126427b.csv", + }, ], columnDescriptions: [ { @@ -258,7 +264,17 @@ module.exports = function (app, port) { shared: Math.random() < 0.8, resultUrls: notExecuted ? [] - : [`/api/results/results.xlsx`, `/api/results/results.csv`], + : [ + { + label: "XLSX", + url: "http://localhost:8080/api/result/xlsx/mimic-iii-demo.51cd95fd-90b2-4573-aab5-11846126427b.xlsx", + }, + { + label: "CSV", + url: "http://localhost:8080/api/result/csv/mimic-iii-demo.51cd95fd-90b2-4573-aab5-11846126427b.csv", + }, + ], + ownerName: "System", ...(Math.random() > 0.2 ? { queryType: "CONCEPT_QUERY" } diff --git a/frontend/mock-api/stored-queries/25.json b/frontend/mock-api/stored-queries/25.json index fe34fef9fd..19503436fc 100644 --- a/frontend/mock-api/stored-queries/25.json +++ b/frontend/mock-api/stored-queries/25.json @@ -2,7 +2,20 @@ "id": 25, "status": "DONE", "numberOfResults": 5, - "resultUrls": ["/api/results/1234-123123123-123123-1231.csv"], + "resultUrls": [ + { + "label": "Alle Dateien", + "url": "http://localhost:8080/api/results/all.zip" + }, + { + "label": "Bericht.pdf", + "url": "http://localhost:8080/api/results/Bericht.pdf" + }, + { + "label": "Statistik.xlsx", + "url": "http://localhost:8080/api/results/Statistik.xlsx" + } + ], "createdAt": "2016-12-02T14:19:09Z", "version": 0, "tags": ["Fun"], diff --git a/frontend/src/js/api/types.ts b/frontend/src/js/api/types.ts index 311659bd91..7d013f5087 100644 --- a/frontend/src/js/api/types.ts +++ b/frontend/src/js/api/types.ts @@ -390,12 +390,17 @@ export interface ColumnDescription { userConceptLabel: string | null; } +export interface ResultUrlsWithLabel { + label: string; + url: string; +} + // TODO: This actually returns GETQueryResponseT => a lot of unused fields export interface GetQueryResponseDoneT { status: "DONE" | "NEW"; // NEW might mean canceled (query not (yet) executed) label: string; numberOfResults: number | null; - resultUrls: string[]; + resultUrls: ResultUrlsWithLabel[]; columnDescriptions: ColumnDescription[] | null; queryType: "CONCEPT_QUERY" | "SECONDARY_ID_QUERY"; requiredTime: number; // In ms, unused at the moment @@ -529,7 +534,7 @@ export interface EntityInfo { } export type GetEntityHistoryResponse = { - resultUrls: string[]; + resultUrls: ResultUrlsWithLabel[]; columnDescriptions: ColumnDescription[]; infos: EntityInfo[]; }; diff --git a/frontend/src/js/button/DownloadButton.tsx b/frontend/src/js/button/DownloadButton.tsx index 00173e566d..c4b136d274 100644 --- a/frontend/src/js/button/DownloadButton.tsx +++ b/frontend/src/js/button/DownloadButton.tsx @@ -1,5 +1,7 @@ import styled from "@emotion/styled"; import { ReactNode, useContext, forwardRef } from "react"; +import { ResultUrlsWithLabel } from "../api/types"; +import { IconName } from "@fortawesome/fontawesome-svg-core"; import { AuthTokenContext } from "../authorization/AuthTokenProvider"; @@ -13,8 +15,31 @@ const Link = styled("a")` line-height: 1; `; +const fileTypeToIcon: Record = { + "ZIP": "file-archive", + "XLSX": "file-excel", + "PDF": "file-pdf", + "CSV": "file-csv", +} +function getFileIcon(label:string): IconName { + // Editor Requests + if(label in fileTypeToIcon) { + return fileTypeToIcon[label]; + } + + // Forms + if(label.includes(".")){ + const ext = label.split(".").pop(); + if(!ext) return "file-download"; + if(ext in fileTypeToIcon){ + return fileTypeToIcon[ext]; + } + } + return "file-download"; +} + interface Props extends Omit { - url: string; + url: ResultUrlsWithLabel; className?: string; children?: ReactNode; onClick?: () => void; @@ -24,15 +49,13 @@ const DownloadButton = forwardRef( ({ url, className, children, onClick, ...restProps }, ref) => { const { authToken } = useContext(AuthTokenContext); - const href = `${url}?access_token=${encodeURIComponent( + const href = `${url.url}?access_token=${encodeURIComponent( authToken, )}&charset=ISO_8859_1`; - const icon = "download"; - return ( - + {children} diff --git a/frontend/src/js/button/PreviewButton.tsx b/frontend/src/js/button/PreviewButton.tsx index 09144a9dec..196c37ba50 100644 --- a/frontend/src/js/button/PreviewButton.tsx +++ b/frontend/src/js/button/PreviewButton.tsx @@ -3,7 +3,7 @@ import { FC } from "react"; import { useTranslation } from "react-i18next"; import { useDispatch } from "react-redux"; -import type { ColumnDescription } from "../api/types"; +import type { ColumnDescription, ResultUrlsWithLabel } from "../api/types"; import { useGetAuthorizedUrl } from "../authorization/useAuthorizedUrl"; import { openPreview, useLoadPreviewData } from "../preview/actions"; @@ -16,7 +16,7 @@ const Button = styled(TransparentButton)` interface PropsT { columns: ColumnDescription[]; - url: string; + url: ResultUrlsWithLabel; } const PreviewButton: FC = ({ url, columns, ...restProps }) => { @@ -29,7 +29,7 @@ const PreviewButton: FC = ({ url, columns, ...restProps }) => { return (