Skip to content

Commit

Permalink
properly format apiPayload for ExternalForm
Browse files Browse the repository at this point in the history
  • Loading branch information
@awildturtok
awildturtok committed Jun 7, 2023
1 parent 56924de commit 649ea9a
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 97 deletions.
13 changes: 10 additions & 3 deletions backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.deser.ContextualDeserializer;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
import com.google.common.base.Strings;
import lombok.AllArgsConstructor;
import lombok.Getter;
Expand All @@ -60,16 +61,22 @@ public class ExternalForm extends Form implements SubTyped {
@JsonValue
@ToString.Exclude
private final ObjectNode node;
private final String subType;

public JsonNode getExternalApiPayload() {
return ((ObjectNode) node.deepCopy()
.without("values"))
.set("type", new TextNode(subType));

}

@Nullable
@Override
@JsonIgnore
public JsonNode getValues() {
return node;
return node.get("values");
}

private final String subType;

@Override
public String getLocalizedTypeLabel() {
final JsonNode formTitle = node.get("title");
Expand Down
11 changes: 6 additions & 5 deletions backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import com.bakdata.conquery.io.cps.CPSType;
import com.bakdata.conquery.io.cps.CPSTypeIdResolver;
import com.bakdata.conquery.io.external.form.ExternalFormBackendApi;
import com.bakdata.conquery.io.external.form.ExternalFormBackendHealthCheck;
import com.bakdata.conquery.io.external.form.ExternalFormMixin;
import com.bakdata.conquery.models.auth.entities.User;
import com.bakdata.conquery.models.auth.permissions.Ability;
Expand Down Expand Up @@ -94,7 +93,9 @@ public void initialize(ManagerNode managerNode) {
client.register(new JacksonMessageBodyProvider(om));

// Register health check
managerNode.getEnvironment().healthChecks().register(getId(), new ExternalFormBackendHealthCheck(createApi()));
final ExternalFormBackendApi externalApi = createApi();

managerNode.getEnvironment().healthChecks().register(getId(), externalApi.createHealthCheck());

// Register form configuration provider
managerNode.getFormScanner().registerFrontendFormConfigProvider(this::registerFormConfigs);
Expand All @@ -121,11 +122,11 @@ public boolean supportsFormType(String formType) {
* @param formConfigs Collection to add received form configs to.
*/
private void registerFormConfigs(ImmutableCollection.Builder<FormFrontendConfigInformation> formConfigs) {
Set<String> supportedFormTypes = new HashSet<>();
final Set<String> supportedFormTypes = new HashSet<>();

for (ObjectNode formConfig : createApi().getFormConfigs()) {
final String subType = formConfig.get("type").asText();
String formType = createSubTypedId(subType);
final String formType = createSubTypedId(subType);

// Override type with our subtype
formConfig.set("type", new TextNode(formType));
Expand Down Expand Up @@ -155,7 +156,7 @@ public User createServiceUser(User originalUser, Dataset dataset) {
// the actual user and download permissions.
final User
serviceUser =
managerNode.getAuthController().flatCopyUser(originalUser, String.format("%s_%s", this.getClass().getSimpleName().toLowerCase(), getId()));
managerNode.getAuthController().flatCopyUser(originalUser, String.format("%s_%s", getClass().getSimpleName().toLowerCase(), getId()));

// The user is able to read the dataset, ensure that the service user can download results
serviceUser.addPermission(dataset.createPermission(Ability.DOWNLOAD.asSet()));
Expand Down
117 changes: 58 additions & 59 deletions backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@ public static void importPermissionConstellation(MetaStorage storage, Role[] rol
}

for (RequiredUser rUser : rUsers) {
User user = rUser.getUser();
final User user = rUser.getUser();
storage.addUser(user);

RoleId[] rolesInjected = rUser.getRolesInjected();
final RoleId[] rolesInjected = rUser.getRolesInjected();

for (RoleId mandatorId : rolesInjected) {
user.addRole(storage.getRole(mandatorId));
Expand All @@ -60,45 +60,6 @@ public static Query parseQuery(StandaloneSupport support, JsonNode rawQuery) thr
return ConqueryTestSpec.parseSubTree(support, rawQuery, Query.class);
}


private static URI getPostQueryURI(StandaloneSupport conquery) {
return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), DatasetQueryResource.class, "postQuery")
.buildFromMap(Map.of(
"dataset", conquery.getDataset().getId()
));
}

private static JsonNode getRawExecutionStatus(String id, StandaloneSupport conquery, User user) {
final URI queryStatusURI = getQueryStatusURI(conquery, id);
// We try at most 5 times, queryStatus waits for 10s, we therefore don't need to timeout here.
// Query getQueryStatus until it is no longer running.
for (int trial = 0; trial < 5; trial++) {
log.debug("Trying to get Query result");

JsonNode execStatusRaw =
conquery.getClient()
.target(queryStatusURI)
.request(MediaType.APPLICATION_JSON_TYPE)
.header("Authorization", "Bearer " + conquery.getAuthorizationController().getConqueryTokenRealm().createTokenForUser(user.getId()))
.get(JsonNode.class);

String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText();

if (!ExecutionState.RUNNING.name().equals(status)) {
return execStatusRaw;
}
}

throw new IllegalStateException("Query was running too long.");
}

private static URI getQueryStatusURI(StandaloneSupport conquery, String id) {
return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "getStatus")
.buildFromMap(Map.of(
"query", id, "dataset", conquery.getDataset().getId()
));
}

/**
* Send a query onto the conquery instance and assert the result's size.
*
Expand All @@ -112,15 +73,15 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O
.createTokenForUser(user.getId());

// Submit Query
Response response = conquery.getClient()
.target(postQueryURI)
.request(MediaType.APPLICATION_JSON_TYPE)
.header("Authorization", "Bearer " + userToken)
.post(Entity.entity(query, MediaType.APPLICATION_JSON_TYPE));
final Response response = conquery.getClient()
.target(postQueryURI)
.request(MediaType.APPLICATION_JSON_TYPE)
.header("Authorization", "Bearer " + userToken)
.post(Entity.entity(query, MediaType.APPLICATION_JSON_TYPE));


assertThat(response.getStatusInfo().getStatusCode()).as("Result of %s", postQueryURI)
.isEqualTo(expectedResponseCode);
.isEqualTo(expectedResponseCode);

if (expectedState == ExecutionState.FAILED && !response.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL)) {
return null;
Expand All @@ -132,14 +93,14 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O

// TODO implement this properly: ExecutionStatus status = response.readEntity(ExecutionStatus.Full.class);

JsonNode execStatusRaw = getRawExecutionStatus(id, conquery, user);
final JsonNode execStatusRaw = getRawExecutionStatus(id, conquery, user);

String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText();
long numberOfResults = execStatusRaw.get(ExecutionStatus.Fields.numberOfResults).asLong(0);
final String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText();
final long numberOfResults = execStatusRaw.get(ExecutionStatus.Fields.numberOfResults).asLong(0);

assertThat(status).isEqualTo(expectedState.name());

if (expectedState == ExecutionState.DONE && expectedSize != -1) {
if (expectedState == ExecutionState.DONE && expectedSize != -1) {
assertThat(numberOfResults)
.describedAs("Query results")
.isEqualTo(expectedSize);
Expand All @@ -148,22 +109,60 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O
return ManagedExecutionId.Parser.INSTANCE.parse(id);
}

private static URI getPostQueryURI(StandaloneSupport conquery) {
return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), DatasetQueryResource.class, "postQuery")
.buildFromMap(Map.of(
"dataset", conquery.getDataset().getId()
));
}

private static JsonNode getRawExecutionStatus(String id, StandaloneSupport conquery, User user) {
final URI queryStatusURI = getQueryStatusURI(conquery, id);
// We try at most 5 times, queryStatus waits for 10s, we therefore don't need to timeout here.
// Query getQueryStatus until it is no longer running.
for (int trial = 0; trial < 5; trial++) {
log.debug("Trying to get Query result");

final JsonNode execStatusRaw =
conquery.getClient()
.target(queryStatusURI)
.request(MediaType.APPLICATION_JSON_TYPE)
.header("Authorization", "Bearer " + conquery.getAuthorizationController().getConqueryTokenRealm().createTokenForUser(user.getId()))
.get(JsonNode.class);

final String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText();

if (!ExecutionState.RUNNING.name().equals(status)) {
return execStatusRaw;
}
}

throw new IllegalStateException("Query was running too long.");
}

private static URI getQueryStatusURI(StandaloneSupport conquery, String id) {
return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "getStatus")
.buildFromMap(Map.of(
"query", id, "dataset", conquery.getDataset().getId()
));
}

public static FullExecutionStatus getExecutionStatus(StandaloneSupport conquery, ManagedExecutionId executionId, User user, int expectedResponseCode) {
final URI queryStatusURI = getQueryStatusURI(conquery, executionId.toString());

final String userToken = conquery.getAuthorizationController()
.getConqueryTokenRealm()
.createTokenForUser(user.getId());
.getConqueryTokenRealm()
.createTokenForUser(user.getId());

Response response = conquery.getClient()
.target(queryStatusURI)
.request(MediaType.APPLICATION_JSON_TYPE)
.header("Authorization", "Bearer " + userToken)
.get();
final Response response = conquery.getClient()
.target(queryStatusURI)
.request(MediaType.APPLICATION_JSON_TYPE)
.header("Authorization", "Bearer " + userToken)
.get();


assertThat(response.getStatusInfo().getStatusCode()).as("Result of %s", queryStatusURI)
.isEqualTo(expectedResponseCode);
.isEqualTo(expectedResponseCode);


return response.readEntity(FullExecutionStatus.class);
Expand Down
44 changes: 14 additions & 30 deletions backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package com.bakdata.conquery.integration.tests;

import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
import static org.mockserver.model.HttpRequest.request;

import java.io.File;
Expand Down Expand Up @@ -34,6 +33,7 @@
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.mockserver.integration.ClientAndServer;
import org.mockserver.mock.Expectation;
import org.mockserver.mock.OpenAPIExpectation;
import org.mockserver.model.HttpResponse;
import org.mockserver.model.StringBody;
Expand All @@ -56,7 +56,8 @@ public void execute(String name, TestConquery testConquery) throws Exception {
.getEnvironment()
.healthChecks()
.runHealthCheck(FORM_BACKEND_ID)
.isHealthy()).describedAs("Checking health of form backend").isTrue();
.isHealthy())
.describedAs("Checking health of form backend").isTrue();

log.info("Get external form configs");
final FormScanner formScanner = testConquery.getStandaloneCommand().getManager().getFormScanner();
Expand All @@ -76,11 +77,8 @@ public void execute(String name, TestConquery testConquery) throws Exception {


// Generate asset urls and check them in the status
final UriBuilder apiUriBuilder = testConquery.getSupport(name)
.defaultApiURIBuilder();
final ManagedExecution storedExecution = testConquery.getSupport(name)
.getMetaStorage()
.getExecution(managedExecutionId);
final UriBuilder apiUriBuilder = testConquery.getSupport(name).defaultApiURIBuilder();
final ManagedExecution storedExecution = testConquery.getSupport(name).getMetaStorage().getExecution(managedExecutionId);
final URI
downloadURLasset1 =
ResultExternalResource.getDownloadURL(apiUriBuilder.clone(), (ManagedExecution & ExternalResult) storedExecution, executionStatus.getResultUrls()
Expand All @@ -94,18 +92,12 @@ public void execute(String name, TestConquery testConquery) throws Exception {


assertThat(executionStatus.getStatus()).isEqualTo(ExecutionState.DONE);
assertThat(executionStatus.getResultUrls()).containsExactly(
new ResultAsset("Result", downloadURLasset1),
new ResultAsset("Another Result", downloadURLasset2)
);
assertThat(executionStatus.getResultUrls()).containsExactly(new ResultAsset("Result", downloadURLasset1), new ResultAsset("Another Result", downloadURLasset2));

log.info("Download Result");
final String
response =
support.getClient()
.target(executionStatus.getResultUrls().get(0).url())
.request(javax.ws.rs.core.MediaType.TEXT_PLAIN_TYPE)
.get(String.class);
support.getClient().target(executionStatus.getResultUrls().get(0).url()).request(javax.ws.rs.core.MediaType.TEXT_PLAIN_TYPE).get(String.class);

assertThat(response).isEqualTo("Hello");

Expand Down Expand Up @@ -139,33 +131,25 @@ public ConqueryConfig overrideConfig(ConqueryConfig conf, File workdir) {
return conf.withStorage(storageConfig.withDirectory(storageDir));
}

@SneakyThrows
@NotNull
private URI createFormServer() throws IOException {
log.info("Starting mock form backend server");
formBackend = ClientAndServer.startClientAndServer(1080);

final URI baseURI = URI.create(String.format("http://127.0.0.1:%d", formBackend.getPort()));

formBackend.upsert(OpenAPIExpectation.openAPIExpectation("/com/bakdata/conquery/external/openapi-form-backend.yaml"));
Expectation[] expectations = formBackend.upsert(OpenAPIExpectation.openAPIExpectation("/com/bakdata/conquery/external/openapi-form-backend.yaml"));


// Result request matcher
formBackend.when(request("/result.txt"))
.respond(HttpResponse.response()
.withBody(StringBody.exact("Hello")
)
);
formBackend.when(request("/result.txt")).respond(HttpResponse.response().withBody(StringBody.exact("Hello")));


// Trap: Log failed request
formBackend.when(request()).respond(httpRequest -> {
log.error(
"{} on {}\n\t Headers: {}\n\tBody {}",
httpRequest.getMethod(),
httpRequest.getPath(),
httpRequest.getHeaderList(),
httpRequest.getBodyAsString()
);
fail("Trapped because request did not match. See log.");
throw new Error("Received unmappable request");
log.error("{} on {}\n\t Headers: {}\n\tBody {}", httpRequest.getMethod(), httpRequest.getPath(), httpRequest.getHeaderList(), httpRequest.getBodyAsString());
return HttpResponse.notFoundResponse();
});
return baseURI;
}
Expand Down

0 comments on commit 649ea9a

Please sign in to comment.