diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java index 49ec7b6ca2..cc5fc0cc8b 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java @@ -37,6 +37,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.deser.ContextualDeserializer; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.base.Strings; import lombok.AllArgsConstructor; import lombok.Getter; @@ -60,16 +61,22 @@ public class ExternalForm extends Form implements SubTyped { @JsonValue @ToString.Exclude private final ObjectNode node; + private final String subType; + + public JsonNode getExternalApiPayload() { + return ((ObjectNode) node.deepCopy() + .without("values")) + .set("type", new TextNode(subType)); + + } @Nullable @Override @JsonIgnore public JsonNode getValues() { - return node; + return node.get("values"); } - private final String subType; - @Override public String getLocalizedTypeLabel() { final JsonNode formTitle = node.get("title"); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java index da0e4f9ae9..d592a8f691 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java @@ -17,7 +17,6 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.external.form.ExternalFormBackendApi; -import com.bakdata.conquery.io.external.form.ExternalFormBackendHealthCheck; import com.bakdata.conquery.io.external.form.ExternalFormMixin; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; @@ -94,7 +93,9 @@ public void initialize(ManagerNode managerNode) { client.register(new JacksonMessageBodyProvider(om)); // Register health check - managerNode.getEnvironment().healthChecks().register(getId(), new ExternalFormBackendHealthCheck(createApi())); + final ExternalFormBackendApi externalApi = createApi(); + + managerNode.getEnvironment().healthChecks().register(getId(), externalApi.createHealthCheck()); // Register form configuration provider managerNode.getFormScanner().registerFrontendFormConfigProvider(this::registerFormConfigs); @@ -121,11 +122,11 @@ public boolean supportsFormType(String formType) { * @param formConfigs Collection to add received form configs to. */ private void registerFormConfigs(ImmutableCollection.Builder formConfigs) { - Set supportedFormTypes = new HashSet<>(); + final Set supportedFormTypes = new HashSet<>(); for (ObjectNode formConfig : createApi().getFormConfigs()) { final String subType = formConfig.get("type").asText(); - String formType = createSubTypedId(subType); + final String formType = createSubTypedId(subType); // Override type with our subtype formConfig.set("type", new TextNode(formType)); @@ -155,7 +156,7 @@ public User createServiceUser(User originalUser, Dataset dataset) { // the actual user and download permissions. final User serviceUser = - managerNode.getAuthController().flatCopyUser(originalUser, String.format("%s_%s", this.getClass().getSimpleName().toLowerCase(), getId())); + managerNode.getAuthController().flatCopyUser(originalUser, String.format("%s_%s", getClass().getSimpleName().toLowerCase(), getId())); // The user is able to read the dataset, ensure that the service user can download results serviceUser.addPermission(dataset.createPermission(Ability.DOWNLOAD.asSet())); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java index be2ccc74e5..5f34d01a92 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java @@ -44,10 +44,10 @@ public static void importPermissionConstellation(MetaStorage storage, Role[] rol } for (RequiredUser rUser : rUsers) { - User user = rUser.getUser(); + final User user = rUser.getUser(); storage.addUser(user); - RoleId[] rolesInjected = rUser.getRolesInjected(); + final RoleId[] rolesInjected = rUser.getRolesInjected(); for (RoleId mandatorId : rolesInjected) { user.addRole(storage.getRole(mandatorId)); @@ -60,45 +60,6 @@ public static Query parseQuery(StandaloneSupport support, JsonNode rawQuery) thr return ConqueryTestSpec.parseSubTree(support, rawQuery, Query.class); } - - private static URI getPostQueryURI(StandaloneSupport conquery) { - return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), DatasetQueryResource.class, "postQuery") - .buildFromMap(Map.of( - "dataset", conquery.getDataset().getId() - )); - } - - private static JsonNode getRawExecutionStatus(String id, StandaloneSupport conquery, User user) { - final URI queryStatusURI = getQueryStatusURI(conquery, id); - // We try at most 5 times, queryStatus waits for 10s, we therefore don't need to timeout here. - // Query getQueryStatus until it is no longer running. - for (int trial = 0; trial < 5; trial++) { - log.debug("Trying to get Query result"); - - JsonNode execStatusRaw = - conquery.getClient() - .target(queryStatusURI) - .request(MediaType.APPLICATION_JSON_TYPE) - .header("Authorization", "Bearer " + conquery.getAuthorizationController().getConqueryTokenRealm().createTokenForUser(user.getId())) - .get(JsonNode.class); - - String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText(); - - if (!ExecutionState.RUNNING.name().equals(status)) { - return execStatusRaw; - } - } - - throw new IllegalStateException("Query was running too long."); - } - - private static URI getQueryStatusURI(StandaloneSupport conquery, String id) { - return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "getStatus") - .buildFromMap(Map.of( - "query", id, "dataset", conquery.getDataset().getId() - )); - } - /** * Send a query onto the conquery instance and assert the result's size. * @@ -112,15 +73,15 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O .createTokenForUser(user.getId()); // Submit Query - Response response = conquery.getClient() - .target(postQueryURI) - .request(MediaType.APPLICATION_JSON_TYPE) - .header("Authorization", "Bearer " + userToken) - .post(Entity.entity(query, MediaType.APPLICATION_JSON_TYPE)); + final Response response = conquery.getClient() + .target(postQueryURI) + .request(MediaType.APPLICATION_JSON_TYPE) + .header("Authorization", "Bearer " + userToken) + .post(Entity.entity(query, MediaType.APPLICATION_JSON_TYPE)); assertThat(response.getStatusInfo().getStatusCode()).as("Result of %s", postQueryURI) - .isEqualTo(expectedResponseCode); + .isEqualTo(expectedResponseCode); if (expectedState == ExecutionState.FAILED && !response.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL)) { return null; @@ -132,14 +93,14 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O // TODO implement this properly: ExecutionStatus status = response.readEntity(ExecutionStatus.Full.class); - JsonNode execStatusRaw = getRawExecutionStatus(id, conquery, user); + final JsonNode execStatusRaw = getRawExecutionStatus(id, conquery, user); - String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText(); - long numberOfResults = execStatusRaw.get(ExecutionStatus.Fields.numberOfResults).asLong(0); + final String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText(); + final long numberOfResults = execStatusRaw.get(ExecutionStatus.Fields.numberOfResults).asLong(0); assertThat(status).isEqualTo(expectedState.name()); - if (expectedState == ExecutionState.DONE && expectedSize != -1) { + if (expectedState == ExecutionState.DONE && expectedSize != -1) { assertThat(numberOfResults) .describedAs("Query results") .isEqualTo(expectedSize); @@ -148,22 +109,60 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O return ManagedExecutionId.Parser.INSTANCE.parse(id); } + private static URI getPostQueryURI(StandaloneSupport conquery) { + return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), DatasetQueryResource.class, "postQuery") + .buildFromMap(Map.of( + "dataset", conquery.getDataset().getId() + )); + } + + private static JsonNode getRawExecutionStatus(String id, StandaloneSupport conquery, User user) { + final URI queryStatusURI = getQueryStatusURI(conquery, id); + // We try at most 5 times, queryStatus waits for 10s, we therefore don't need to timeout here. + // Query getQueryStatus until it is no longer running. + for (int trial = 0; trial < 5; trial++) { + log.debug("Trying to get Query result"); + + final JsonNode execStatusRaw = + conquery.getClient() + .target(queryStatusURI) + .request(MediaType.APPLICATION_JSON_TYPE) + .header("Authorization", "Bearer " + conquery.getAuthorizationController().getConqueryTokenRealm().createTokenForUser(user.getId())) + .get(JsonNode.class); + + final String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText(); + + if (!ExecutionState.RUNNING.name().equals(status)) { + return execStatusRaw; + } + } + + throw new IllegalStateException("Query was running too long."); + } + + private static URI getQueryStatusURI(StandaloneSupport conquery, String id) { + return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "getStatus") + .buildFromMap(Map.of( + "query", id, "dataset", conquery.getDataset().getId() + )); + } + public static FullExecutionStatus getExecutionStatus(StandaloneSupport conquery, ManagedExecutionId executionId, User user, int expectedResponseCode) { final URI queryStatusURI = getQueryStatusURI(conquery, executionId.toString()); final String userToken = conquery.getAuthorizationController() - .getConqueryTokenRealm() - .createTokenForUser(user.getId()); + .getConqueryTokenRealm() + .createTokenForUser(user.getId()); - Response response = conquery.getClient() - .target(queryStatusURI) - .request(MediaType.APPLICATION_JSON_TYPE) - .header("Authorization", "Bearer " + userToken) - .get(); + final Response response = conquery.getClient() + .target(queryStatusURI) + .request(MediaType.APPLICATION_JSON_TYPE) + .header("Authorization", "Bearer " + userToken) + .get(); assertThat(response.getStatusInfo().getStatusCode()).as("Result of %s", queryStatusURI) - .isEqualTo(expectedResponseCode); + .isEqualTo(expectedResponseCode); return response.readEntity(FullExecutionStatus.class); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java index 58ed118406..3931d12eeb 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.integration.tests; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; import static org.mockserver.model.HttpRequest.request; import java.io.File; @@ -34,6 +33,7 @@ import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import org.mockserver.integration.ClientAndServer; +import org.mockserver.mock.Expectation; import org.mockserver.mock.OpenAPIExpectation; import org.mockserver.model.HttpResponse; import org.mockserver.model.StringBody; @@ -56,7 +56,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { .getEnvironment() .healthChecks() .runHealthCheck(FORM_BACKEND_ID) - .isHealthy()).describedAs("Checking health of form backend").isTrue(); + .isHealthy()) + .describedAs("Checking health of form backend").isTrue(); log.info("Get external form configs"); final FormScanner formScanner = testConquery.getStandaloneCommand().getManager().getFormScanner(); @@ -76,11 +77,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Generate asset urls and check them in the status - final UriBuilder apiUriBuilder = testConquery.getSupport(name) - .defaultApiURIBuilder(); - final ManagedExecution storedExecution = testConquery.getSupport(name) - .getMetaStorage() - .getExecution(managedExecutionId); + final UriBuilder apiUriBuilder = testConquery.getSupport(name).defaultApiURIBuilder(); + final ManagedExecution storedExecution = testConquery.getSupport(name).getMetaStorage().getExecution(managedExecutionId); final URI downloadURLasset1 = ResultExternalResource.getDownloadURL(apiUriBuilder.clone(), (ManagedExecution & ExternalResult) storedExecution, executionStatus.getResultUrls() @@ -94,18 +92,12 @@ public void execute(String name, TestConquery testConquery) throws Exception { assertThat(executionStatus.getStatus()).isEqualTo(ExecutionState.DONE); - assertThat(executionStatus.getResultUrls()).containsExactly( - new ResultAsset("Result", downloadURLasset1), - new ResultAsset("Another Result", downloadURLasset2) - ); + assertThat(executionStatus.getResultUrls()).containsExactly(new ResultAsset("Result", downloadURLasset1), new ResultAsset("Another Result", downloadURLasset2)); log.info("Download Result"); final String response = - support.getClient() - .target(executionStatus.getResultUrls().get(0).url()) - .request(javax.ws.rs.core.MediaType.TEXT_PLAIN_TYPE) - .get(String.class); + support.getClient().target(executionStatus.getResultUrls().get(0).url()).request(javax.ws.rs.core.MediaType.TEXT_PLAIN_TYPE).get(String.class); assertThat(response).isEqualTo("Hello"); @@ -139,6 +131,7 @@ public ConqueryConfig overrideConfig(ConqueryConfig conf, File workdir) { return conf.withStorage(storageConfig.withDirectory(storageDir)); } + @SneakyThrows @NotNull private URI createFormServer() throws IOException { log.info("Starting mock form backend server"); @@ -146,26 +139,17 @@ private URI createFormServer() throws IOException { final URI baseURI = URI.create(String.format("http://127.0.0.1:%d", formBackend.getPort())); - formBackend.upsert(OpenAPIExpectation.openAPIExpectation("/com/bakdata/conquery/external/openapi-form-backend.yaml")); + Expectation[] expectations = formBackend.upsert(OpenAPIExpectation.openAPIExpectation("/com/bakdata/conquery/external/openapi-form-backend.yaml")); + // Result request matcher - formBackend.when(request("/result.txt")) - .respond(HttpResponse.response() - .withBody(StringBody.exact("Hello") - ) - ); + formBackend.when(request("/result.txt")).respond(HttpResponse.response().withBody(StringBody.exact("Hello"))); + // Trap: Log failed request formBackend.when(request()).respond(httpRequest -> { - log.error( - "{} on {}\n\t Headers: {}\n\tBody {}", - httpRequest.getMethod(), - httpRequest.getPath(), - httpRequest.getHeaderList(), - httpRequest.getBodyAsString() - ); - fail("Trapped because request did not match. See log."); - throw new Error("Received unmappable request"); + log.error("{} on {}\n\t Headers: {}\n\tBody {}", httpRequest.getMethod(), httpRequest.getPath(), httpRequest.getHeaderList(), httpRequest.getBodyAsString()); + return HttpResponse.notFoundResponse(); }); return baseURI; }