Skip to content

Commit

Permalink
Merge pull request #3087 from ingef/fix/seralization-of-external-form
Browse files Browse the repository at this point in the history 
Fixes which values are sent to Frontend and external Api
  • Loading branch information
@awildturtok
awildturtok authored Jun 7, 2023
2 parents eb18de5 + 649ea9a commit ab19e25
Show file tree
Hide file tree
Showing 6 changed files with 96 additions and 122 deletions.
13 changes: 10 additions & 3 deletions backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.deser.ContextualDeserializer;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
import com.google.common.base.Strings;
import lombok.AllArgsConstructor;
import lombok.Getter;
Expand All @@ -60,16 +61,22 @@ public class ExternalForm extends Form implements SubTyped {
@JsonValue
@ToString.Exclude
private final ObjectNode node;
private final String subType;

public JsonNode getExternalApiPayload() {
return ((ObjectNode) node.deepCopy()
.without("values"))
.set("type", new TextNode(subType));

}

@Nullable
@Override
@JsonIgnore
public JsonNode getValues() {
return node;
return node.get("values");
}

private final String subType;

@Override
public String getLocalizedTypeLabel() {
final JsonNode formTitle = node.get("title");
Expand Down
18 changes: 8 additions & 10 deletions backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.bakdata.conquery.models.datasets.Dataset;
import com.codahale.metrics.health.HealthCheck;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.dropwizard.health.check.http.HttpHealthCheck;
import lombok.extern.slf4j.Slf4j;

@Slf4j
Expand Down Expand Up @@ -96,7 +97,8 @@ public ExternalTaskState postForm(ExternalForm form, User originalUser, User ser
.header(HTTP_HEADER_CQ_AUTHENTICATION, serviceUserToken)
.header(HTTP_HEADER_CQ_AUTHENTICATION_ORIGINAL, originalUserToken);

return request.post(Entity.entity(form, MediaType.APPLICATION_JSON_TYPE), ExternalTaskState.class);
ExternalTaskState post = request.post(Entity.entity(form.getExternalApiPayload(), MediaType.APPLICATION_JSON_TYPE), ExternalTaskState.class);
return post;
}

public ExternalTaskState getFormState(UUID externalId) {
Expand All @@ -113,14 +115,10 @@ public Response getResult(final URI resultURL) {

}

public HealthCheck.Result checkHealth() {
log.trace("Checking health from: {}", getHealthTarget);
try {
getHealthTarget.request(MediaType.APPLICATION_JSON_TYPE).get(Void.class);
return HealthCheck.Result.healthy();
}
catch (Exception e) {
return HealthCheck.Result.unhealthy(e.getMessage());
}
public HealthCheck createHealthCheck() {
return new HttpHealthCheck(
getHealthTarget.getUri().toString(), client
);
}

}
15 changes: 0 additions & 15 deletions ...d/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendHealthCheck.java
View file

This file was deleted.

11 changes: 6 additions & 5 deletions backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import com.bakdata.conquery.io.cps.CPSType;
import com.bakdata.conquery.io.cps.CPSTypeIdResolver;
import com.bakdata.conquery.io.external.form.ExternalFormBackendApi;
import com.bakdata.conquery.io.external.form.ExternalFormBackendHealthCheck;
import com.bakdata.conquery.io.external.form.ExternalFormMixin;
import com.bakdata.conquery.models.auth.entities.User;
import com.bakdata.conquery.models.auth.permissions.Ability;
Expand Down Expand Up @@ -94,7 +93,9 @@ public void initialize(ManagerNode managerNode) {
client.register(new JacksonMessageBodyProvider(om));

// Register health check
managerNode.getEnvironment().healthChecks().register(getId(), new ExternalFormBackendHealthCheck(createApi()));
final ExternalFormBackendApi externalApi = createApi();

managerNode.getEnvironment().healthChecks().register(getId(), externalApi.createHealthCheck());

// Register form configuration provider
managerNode.getFormScanner().registerFrontendFormConfigProvider(this::registerFormConfigs);
Expand All @@ -121,11 +122,11 @@ public boolean supportsFormType(String formType) {
* @param formConfigs Collection to add received form configs to.
*/
private void registerFormConfigs(ImmutableCollection.Builder<FormFrontendConfigInformation> formConfigs) {
Set<String> supportedFormTypes = new HashSet<>();
final Set<String> supportedFormTypes = new HashSet<>();

for (ObjectNode formConfig : createApi().getFormConfigs()) {
final String subType = formConfig.get("type").asText();
String formType = createSubTypedId(subType);
final String formType = createSubTypedId(subType);

// Override type with our subtype
formConfig.set("type", new TextNode(formType));
Expand Down Expand Up @@ -155,7 +156,7 @@ public User createServiceUser(User originalUser, Dataset dataset) {
// the actual user and download permissions.
final User
serviceUser =
managerNode.getAuthController().flatCopyUser(originalUser, String.format("%s_%s", this.getClass().getSimpleName().toLowerCase(), getId()));
managerNode.getAuthController().flatCopyUser(originalUser, String.format("%s_%s", getClass().getSimpleName().toLowerCase(), getId()));

// The user is able to read the dataset, ensure that the service user can download results
serviceUser.addPermission(dataset.createPermission(Ability.DOWNLOAD.asSet()));
Expand Down
117 changes: 58 additions & 59 deletions backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@ public static void importPermissionConstellation(MetaStorage storage, Role[] rol
}

for (RequiredUser rUser : rUsers) {
User user = rUser.getUser();
final User user = rUser.getUser();
storage.addUser(user);

RoleId[] rolesInjected = rUser.getRolesInjected();
final RoleId[] rolesInjected = rUser.getRolesInjected();

for (RoleId mandatorId : rolesInjected) {
user.addRole(storage.getRole(mandatorId));
Expand All @@ -60,45 +60,6 @@ public static Query parseQuery(StandaloneSupport support, JsonNode rawQuery) thr
return ConqueryTestSpec.parseSubTree(support, rawQuery, Query.class);
}


private static URI getPostQueryURI(StandaloneSupport conquery) {
return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), DatasetQueryResource.class, "postQuery")
.buildFromMap(Map.of(
"dataset", conquery.getDataset().getId()
));
}

private static JsonNode getRawExecutionStatus(String id, StandaloneSupport conquery, User user) {
final URI queryStatusURI = getQueryStatusURI(conquery, id);
// We try at most 5 times, queryStatus waits for 10s, we therefore don't need to timeout here.
// Query getQueryStatus until it is no longer running.
for (int trial = 0; trial < 5; trial++) {
log.debug("Trying to get Query result");

JsonNode execStatusRaw =
conquery.getClient()
.target(queryStatusURI)
.request(MediaType.APPLICATION_JSON_TYPE)
.header("Authorization", "Bearer " + conquery.getAuthorizationController().getConqueryTokenRealm().createTokenForUser(user.getId()))
.get(JsonNode.class);

String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText();

if (!ExecutionState.RUNNING.name().equals(status)) {
return execStatusRaw;
}
}

throw new IllegalStateException("Query was running too long.");
}

private static URI getQueryStatusURI(StandaloneSupport conquery, String id) {
return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "getStatus")
.buildFromMap(Map.of(
"query", id, "dataset", conquery.getDataset().getId()
));
}

/**
* Send a query onto the conquery instance and assert the result's size.
*
Expand All @@ -112,15 +73,15 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O
.createTokenForUser(user.getId());

// Submit Query
Response response = conquery.getClient()
.target(postQueryURI)
.request(MediaType.APPLICATION_JSON_TYPE)
.header("Authorization", "Bearer " + userToken)
.post(Entity.entity(query, MediaType.APPLICATION_JSON_TYPE));
final Response response = conquery.getClient()
.target(postQueryURI)
.request(MediaType.APPLICATION_JSON_TYPE)
.header("Authorization", "Bearer " + userToken)
.post(Entity.entity(query, MediaType.APPLICATION_JSON_TYPE));


assertThat(response.getStatusInfo().getStatusCode()).as("Result of %s", postQueryURI)
.isEqualTo(expectedResponseCode);
.isEqualTo(expectedResponseCode);

if (expectedState == ExecutionState.FAILED && !response.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL)) {
return null;
Expand All @@ -132,14 +93,14 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O

// TODO implement this properly: ExecutionStatus status = response.readEntity(ExecutionStatus.Full.class);

JsonNode execStatusRaw = getRawExecutionStatus(id, conquery, user);
final JsonNode execStatusRaw = getRawExecutionStatus(id, conquery, user);

String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText();
long numberOfResults = execStatusRaw.get(ExecutionStatus.Fields.numberOfResults).asLong(0);
final String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText();
final long numberOfResults = execStatusRaw.get(ExecutionStatus.Fields.numberOfResults).asLong(0);

assertThat(status).isEqualTo(expectedState.name());

if (expectedState == ExecutionState.DONE && expectedSize != -1) {
if (expectedState == ExecutionState.DONE && expectedSize != -1) {
assertThat(numberOfResults)
.describedAs("Query results")
.isEqualTo(expectedSize);
Expand All @@ -148,22 +109,60 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O
return ManagedExecutionId.Parser.INSTANCE.parse(id);
}

private static URI getPostQueryURI(StandaloneSupport conquery) {
return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), DatasetQueryResource.class, "postQuery")
.buildFromMap(Map.of(
"dataset", conquery.getDataset().getId()
));
}

private static JsonNode getRawExecutionStatus(String id, StandaloneSupport conquery, User user) {
final URI queryStatusURI = getQueryStatusURI(conquery, id);
// We try at most 5 times, queryStatus waits for 10s, we therefore don't need to timeout here.
// Query getQueryStatus until it is no longer running.
for (int trial = 0; trial < 5; trial++) {
log.debug("Trying to get Query result");

final JsonNode execStatusRaw =
conquery.getClient()
.target(queryStatusURI)
.request(MediaType.APPLICATION_JSON_TYPE)
.header("Authorization", "Bearer " + conquery.getAuthorizationController().getConqueryTokenRealm().createTokenForUser(user.getId()))
.get(JsonNode.class);

final String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText();

if (!ExecutionState.RUNNING.name().equals(status)) {
return execStatusRaw;
}
}

throw new IllegalStateException("Query was running too long.");
}

private static URI getQueryStatusURI(StandaloneSupport conquery, String id) {
return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "getStatus")
.buildFromMap(Map.of(
"query", id, "dataset", conquery.getDataset().getId()
));
}

public static FullExecutionStatus getExecutionStatus(StandaloneSupport conquery, ManagedExecutionId executionId, User user, int expectedResponseCode) {
final URI queryStatusURI = getQueryStatusURI(conquery, executionId.toString());

final String userToken = conquery.getAuthorizationController()
.getConqueryTokenRealm()
.createTokenForUser(user.getId());
.getConqueryTokenRealm()
.createTokenForUser(user.getId());

Response response = conquery.getClient()
.target(queryStatusURI)
.request(MediaType.APPLICATION_JSON_TYPE)
.header("Authorization", "Bearer " + userToken)
.get();
final Response response = conquery.getClient()
.target(queryStatusURI)
.request(MediaType.APPLICATION_JSON_TYPE)
.header("Authorization", "Bearer " + userToken)
.get();


assertThat(response.getStatusInfo().getStatusCode()).as("Result of %s", queryStatusURI)
.isEqualTo(expectedResponseCode);
.isEqualTo(expectedResponseCode);


return response.readEntity(FullExecutionStatus.class);
Expand Down
Loading

0 comments on commit ab19e25

Please sign in to comment.