Merge pull request #2942 from ingef/release

Merge Release
ingef · Mar 13, 2023 · 81973d3 · 81973d3
2 parents b922dbd + b3ae069
commit 81973d3
Show file tree

Hide file tree

Showing 221 changed files with 10,355 additions and 12,950 deletions.
diff --git a/.github/workflows/lint_openapi.yml b/.github/workflows/lint_openapi.yml
@@ -0,0 +1,15 @@
+name: "OpenAPI linter"
+on:
+  pull_request:
+    paths:
+      - "openapi.yaml"
+      - ".github/workflows/lint_openapi.yaml"
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    container: dshanley/vacuum
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: "Lint OpenApi spec"
+        run: vacuum lint --details ./openapi.yaml
diff --git a/.github/workflows/test_cypress.yml b/.github/workflows/test_cypress.yml
@@ -18,10 +18,15 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: true
 
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
       - name: Set up JDK
         uses: actions/setup-java@v1
         with:

diff --git a/.github/workflows/test_frontend.yml b/.github/workflows/test_frontend.yml
@@ -9,15 +9,12 @@ jobs:
   format-lint-unittest:
     runs-on: ubuntu-latest
     timeout-minutes: 6
-    strategy:
-      matrix:
-        node-version: [16.x]
     steps:
-      - uses: actions/checkout@v2
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
+      - uses: actions/checkout@v3
+      - name: Setup Node
+        uses: actions/setup-node@v3
         with:
-          node-version: ${{ matrix.node-version }}
+          node-version: 18
       - name: install dependencies
         working-directory: ./frontend
         run: yarn --ignore-platform

diff --git a/README.md b/README.md
@@ -17,7 +17,7 @@ Conquery supplies a powerful interface to group event types in a hierarchical *c
 ## Requirements
 - Maven 3 (optional for building)
 - Java JDK 11
-- Node.js 16 + Yarn
+- Node.js 18 + Yarn
 - curl (to import the test data)
 
 

diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java
@@ -1,22 +1,16 @@
 package com.bakdata.conquery.apiv1.forms;
 
-import java.util.List;
-import java.util.Map;
-
 import javax.annotation.Nullable;
 
 import com.bakdata.conquery.apiv1.query.QueryDescription;
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.io.storage.MetaStorage;
-import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.auth.entities.Subject;
 import com.bakdata.conquery.models.auth.permissions.Ability;
 import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner;
 import com.bakdata.conquery.models.forms.managed.ManagedForm;
-import com.bakdata.conquery.models.query.ManagedQuery;
 import com.bakdata.conquery.models.query.visitor.QueryVisitor;
-import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ClassToInstanceMap;
@@ -43,7 +37,6 @@ public String getFormType() {
 		return this.getClass().getAnnotation(CPSType.class).id();
 	}
 
-	public abstract Map<String, List<ManagedQuery>> createSubQueries(DatasetRegistry datasets, User user, Dataset submittedDataset);
 
 
 	@Override

diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/InternalForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/InternalForm.java
@@ -0,0 +1,17 @@
+package com.bakdata.conquery.apiv1.forms;
+
+import java.util.Map;
+
+import com.bakdata.conquery.apiv1.query.Query;
+
+/**
+ * Interface for {@link Form}s that are executed as a {@link com.bakdata.conquery.models.forms.managed.ManagedInternalForm}.
+ *
+ * {@link com.bakdata.conquery.models.forms.managed.ManagedInternalForm}s can consist of multiple sub queries which are generated from the api object.
+ */
+public interface InternalForm {
+
+
+	Map<String, Query> createSubQueries();
+
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java
@@ -11,15 +11,12 @@
 import com.bakdata.conquery.apiv1.query.Query;
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.io.jackson.View;
-import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.common.Range;
-import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery;
 import com.bakdata.conquery.models.forms.util.Alignment;
 import com.bakdata.conquery.models.query.DateAggregationMode;
 import com.bakdata.conquery.models.query.QueryResolveContext;
 import com.bakdata.conquery.models.query.Visitable;
-import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.fasterxml.jackson.annotation.JsonView;
 import lombok.Getter;
 import lombok.Setter;
@@ -41,7 +38,7 @@ public class AbsoluteMode extends Mode {
 	private ArrayConceptQuery resolvedFeatures;
 
 	@Override
-	public Query createSpecializedQuery(DatasetRegistry datasets, User user, Dataset submittedDataset) {
+	public Query createSpecializedQuery() {
 
 		List<ExportForm.ResolutionAndAlignment> resolutionsAndAlignments =
 				ExportForm.getResolutionAlignmentMap(getForm().getResolvedResolutions(), getAlignmentHint());

diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/EntityDateMode.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/EntityDateMode.java
@@ -1,32 +1,24 @@
 package com.bakdata.conquery.apiv1.forms.export_form;
 
 import java.time.LocalDate;
-import java.util.List;
 import java.util.function.Consumer;
 
 import javax.annotation.CheckForNull;
 import javax.validation.Valid;
-import javax.validation.constraints.NotEmpty;
 import javax.validation.constraints.NotNull;
 
 import com.bakdata.conquery.apiv1.query.ArrayConceptQuery;
-import com.bakdata.conquery.apiv1.query.CQElement;
 import com.bakdata.conquery.apiv1.query.Query;
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.io.jackson.View;
-import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.common.Range;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
-import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.forms.managed.EntityDateQuery;
 import com.bakdata.conquery.models.forms.util.Alignment;
 import com.bakdata.conquery.models.query.DateAggregationMode;
 import com.bakdata.conquery.models.query.QueryResolveContext;
 import com.bakdata.conquery.models.query.Visitable;
-import com.bakdata.conquery.models.worker.DatasetRegistry;
-import com.fasterxml.jackson.annotation.JsonBackReference;
 import com.fasterxml.jackson.annotation.JsonView;
-import com.google.common.collect.ImmutableList;
 import lombok.Getter;
 import lombok.Setter;
 
@@ -57,18 +49,18 @@ public void visit(Consumer<Visitable> visitor) {
 	public void resolve(QueryResolveContext context) {
 		resolvedFeatures = ArrayConceptQuery.createFromFeatures(getForm().getFeatures());
 		resolvedFeatures.resolve(context);
-	}
+    }
 
 	@Override
-	public Query createSpecializedQuery(DatasetRegistry datasets, User user, Dataset submittedDataset) {
+	public Query createSpecializedQuery() {
 		CDateRange dateRestriction = dateRange == null ? CDateRange.all() : CDateRange.of(dateRange);
 
-        return new EntityDateQuery(
+		return new EntityDateQuery(
 				getForm().getPrerequisite(),
 				resolvedFeatures,
 				ExportForm.getResolutionAlignmentMap(getForm().getResolvedResolutions(), getAlignmentHint()),
 				dateRestriction,
 				dateAggregationMode
 		);
-    }
+	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java
@@ -1,6 +1,5 @@
 package com.bakdata.conquery.apiv1.forms.export_form;
 
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -15,42 +14,39 @@
 import c10n.C10N;
 import com.bakdata.conquery.ConqueryConstants;
 import com.bakdata.conquery.apiv1.forms.Form;
-import com.bakdata.conquery.apiv1.query.ArrayConceptQuery;
+import com.bakdata.conquery.apiv1.forms.InternalForm;
 import com.bakdata.conquery.apiv1.query.CQElement;
 import com.bakdata.conquery.apiv1.query.Query;
 import com.bakdata.conquery.apiv1.query.QueryDescription;
 import com.bakdata.conquery.internationalization.ExportFormC10n;
 import com.bakdata.conquery.io.cps.CPSType;
-import com.bakdata.conquery.io.jackson.View;
+import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.datasets.Dataset;
-import com.bakdata.conquery.models.execution.ManagedExecution;
 import com.bakdata.conquery.models.forms.managed.ManagedForm;
 import com.bakdata.conquery.models.forms.managed.ManagedInternalForm;
 import com.bakdata.conquery.models.forms.util.Alignment;
 import com.bakdata.conquery.models.forms.util.Resolution;
 import com.bakdata.conquery.models.forms.util.ResolutionShortNames;
 import com.bakdata.conquery.models.i18n.I18n;
 import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
-import com.bakdata.conquery.models.query.DateAggregationMode;
 import com.bakdata.conquery.models.query.ManagedQuery;
 import com.bakdata.conquery.models.query.QueryResolveContext;
 import com.bakdata.conquery.models.query.Visitable;
-import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonManagedReference;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonView;
 import com.google.common.collect.ImmutableList;
 import lombok.AccessLevel;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.Setter;
 
-@Getter @Setter
-@CPSType(id="EXPORT_FORM", base=QueryDescription.class)
-public class ExportForm extends Form {
+@Getter
+@Setter
+@CPSType(id = "EXPORT_FORM", base = QueryDescription.class)
+public class ExportForm extends Form implements InternalForm {
 
 	@NotNull
 	@JsonProperty("queryGroup")
@@ -77,7 +73,6 @@ public class ExportForm extends Form {
 	private Query prerequisite;
 	@JsonIgnore
 	private List<Resolution> resolvedResolutions;
-
 	@Override
 	public void visit(Consumer<Visitable> visitor) {
 		visitor.accept(this);
@@ -87,12 +82,11 @@ public void visit(Consumer<Visitable> visitor) {
 
 
 	@Override
-	public Map<String, List<ManagedQuery>> createSubQueries(DatasetRegistry datasets, User user, Dataset submittedDataset) {
+	public Map<String, Query> createSubQueries() {
 		return Map.of(
-			ConqueryConstants.SINGLE_RESULT_TABLE_NAME,
-			List.of(
-				timeMode.createSpecializedQuery(datasets, user, submittedDataset)
-					.toManagedExecution(user, submittedDataset)));
+				ConqueryConstants.SINGLE_RESULT_TABLE_NAME,
+				timeMode.createSpecializedQuery()
+		);
 	}
 
 	@Override
@@ -102,7 +96,7 @@ public Set<ManagedExecutionId> collectRequiredQueries() {
 
 	@Override
 	public void resolve(QueryResolveContext context) {
-		queryGroup = (ManagedQuery) context.getDatasetRegistry().getMetaRegistry().resolve(queryGroupId);
+		queryGroup = (ManagedQuery) context.getStorage().getExecution(queryGroupId);
 
 
 		// Apply defaults to user concept
@@ -193,7 +187,7 @@ public static void enable(List<CQElement> features) {
 
 
 	@Override
-	public ManagedForm toManagedExecution(User user, Dataset submittedDataset) {
-		return new ManagedInternalForm(this, user, submittedDataset);
+	public ManagedForm toManagedExecution(User user, Dataset submittedDataset, MetaStorage storage) {
+		return new ManagedInternalForm(this, user, submittedDataset, storage);
 	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java
@@ -1,7 +1,6 @@
 package com.bakdata.conquery.apiv1.forms.export_form;
 
 import java.time.LocalDate;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -15,23 +14,23 @@
 import c10n.C10N;
 import com.bakdata.conquery.ConqueryConstants;
 import com.bakdata.conquery.apiv1.forms.Form;
+import com.bakdata.conquery.apiv1.forms.InternalForm;
+import com.bakdata.conquery.apiv1.query.Query;
 import com.bakdata.conquery.apiv1.query.QueryDescription;
 import com.bakdata.conquery.apiv1.query.TableExportQuery;
 import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept;
 import com.bakdata.conquery.internationalization.ExportFormC10n;
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.common.Range;
 import com.bakdata.conquery.models.datasets.Dataset;
-import com.bakdata.conquery.models.execution.ManagedExecution;
-import com.bakdata.conquery.models.forms.managed.ManagedForm;
 import com.bakdata.conquery.models.forms.managed.ManagedInternalForm;
 import com.bakdata.conquery.models.i18n.I18n;
 import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
 import com.bakdata.conquery.models.query.ManagedQuery;
 import com.bakdata.conquery.models.query.QueryResolveContext;
 import com.bakdata.conquery.models.query.Visitable;
-import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.ImmutableList;
@@ -41,7 +40,7 @@
 @Getter
 @Setter
 @CPSType(id = "FULL_EXPORT_FORM", base = QueryDescription.class)
-public class FullExportForm extends Form {
+public class FullExportForm extends Form implements InternalForm {
 
 	@NotNull
 	@JsonProperty("queryGroup")
@@ -65,7 +64,7 @@ public void visit(Consumer<Visitable> visitor) {
 
 
 	@Override
-	public Map<String, List<ManagedQuery>> createSubQueries(DatasetRegistry datasets, User user, Dataset submittedDataset) {
+	public Map<String, Query> createSubQueries() {
 
 		// Forms are sent as an array of standard queries containing AND/OR of CQConcepts, we ignore everything and just convert the CQConcepts into CQUnfiltered for export.
 
@@ -74,12 +73,10 @@ public Map<String, List<ManagedQuery>> createSubQueries(DatasetRegistry datasets
 
 		exportQuery.setTables(tables);
 
-		final ManagedQuery managedQuery = new ManagedQuery(exportQuery, user, submittedDataset);
-
 
 		return Map.of(
 				ConqueryConstants.SINGLE_RESULT_TABLE_NAME,
-				List.of(managedQuery)
+				exportQuery
 		);
 
 	}
@@ -91,7 +88,7 @@ public Set<ManagedExecutionId> collectRequiredQueries() {
 
 	@Override
 	public void resolve(QueryResolveContext context) {
-		queryGroup = (ManagedQuery) context.getDatasetRegistry().getMetaRegistry().resolve(queryGroupId);
+		queryGroup = (ManagedQuery) context.getStorage().getExecution(queryGroupId);
 	}
 
 	@Override
@@ -101,7 +98,7 @@ public String getLocalizedTypeLabel() {
 
 
 	@Override
-	public ManagedForm toManagedExecution(User user, Dataset submittedDataset) {
-		return new ManagedInternalForm(this, user, submittedDataset);
+	public ManagedInternalForm<FullExportForm> toManagedExecution(User user, Dataset submittedDataset, MetaStorage storage) {
+		return new ManagedInternalForm<FullExportForm>(this, user, submittedDataset, storage);
 	}
 }