Merge pull request #3146 from ingef/release

Merge Release
ingef · Aug 15, 2023 · 9db4e42 · 9db4e42
2 parents ab16837 + 61ab823
commit 9db4e42
Show file tree

Hide file tree

Showing 248 changed files with 7,587 additions and 1,503 deletions.
diff --git a/.github/workflows/test_backend.yml b/.github/workflows/test_backend.yml
@@ -14,7 +14,7 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 15
     steps:
       - name: Cache local Maven repository
         uses: actions/cache@v2
@@ -35,8 +35,11 @@ jobs:
       - name: Build Backend
         run: mvn -T 1C install -pl backend -DskipTests -am
       - name: Unit Test
-        run: mvn test -T 1C -pl backend -DexcludedGroups="INTEGRATION_PROGRAMMATIC, INTEGRATION_JSON"
+        run: mvn test -T 1C -pl backend -DexcludedGroups="INTEGRATION_PROGRAMMATIC, INTEGRATION_JSON, INTEGRATION_SQL_BACKEND"
       - name: Programmatic Integration Tests
         run: mvn test -T 1C -pl backend -Dgroups="INTEGRATION_PROGRAMMATIC"
       - name: JSON based Integration Tests
         run: mvn test -T 1C -pl backend -Dgroups="INTEGRATION_JSON"
+      - name: SQL based Integration Tests
+        if: ${{ startsWith(github.head_ref, 'sql/') }}
+        run: mvn test -T 1C -pl backend -Dgroups="INTEGRATION_SQL_BACKEND"
diff --git a/backend/pom.xml b/backend/pom.xml
@@ -346,5 +346,38 @@
             <artifactId>auto-service</artifactId>
             <version>1.0.1</version>
         </dependency>
+        <dependency>
+            <groupId>org.jooq</groupId>
+            <artifactId>jooq</artifactId>
+            <version>3.18.3</version>
+        </dependency>
+        <dependency>
+            <groupId>org.jooq</groupId>
+            <artifactId>jooq-postgres-extensions</artifactId>
+            <version>3.18.3</version>
+        </dependency>
+        <dependency>
+            <groupId>com.zaxxer</groupId>
+            <artifactId>HikariCP</artifactId>
+            <version>5.0.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
+            <version>1.17.6</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <version>1.17.6</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>postgresql</artifactId>
+            <version>1.17.6</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>
diff --git a/backend/src/main/java/com/bakdata/conquery/Conquery.java b/backend/src/main/java/com/bakdata/conquery/Conquery.java
@@ -12,6 +12,10 @@
 import com.bakdata.conquery.commands.StandaloneCommand;
 import com.bakdata.conquery.io.jackson.Jackson;
 import com.bakdata.conquery.io.jackson.MutableInjectableValues;
+import com.bakdata.conquery.mode.Manager;
+import com.bakdata.conquery.mode.ManagerProvider;
+import com.bakdata.conquery.mode.cluster.ClusterManagerProvider;
+import com.bakdata.conquery.mode.local.LocalManagerProvider;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.dropwizard.Application;
@@ -34,7 +38,7 @@ public class Conquery extends Application<ConqueryConfig> {
 
 	private final String name;
 	@Setter
-	private ManagerNode manager;
+	private ManagerNode managerNode;
 
 	public Conquery() {
 		this("Conquery");
@@ -90,10 +94,16 @@ protected Level bootstrapLogLevel() {
 
 	@Override
 	public void run(ConqueryConfig configuration, Environment environment) throws Exception {
-		if (manager == null) {
-			manager = new ManagerNode();
+		ManagerProvider provider = configuration.getSqlConnectorConfig().isEnabled() ?
+								   new LocalManagerProvider() : new ClusterManagerProvider();
+		run(provider.provideManager(configuration, environment));
+	}
+
+	public void run(Manager manager) throws InterruptedException {
+		if (managerNode == null) {
+			managerNode = new ManagerNode();
 		}
-		manager.run(configuration, environment);
+		managerNode.run(manager);
 	}
 
 	public static void main(String... args) throws Exception {

diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/MeProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/MeProcessor.java
@@ -15,6 +15,7 @@
 import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
 import com.bakdata.conquery.models.identifiable.ids.specific.GroupId;
 import com.bakdata.conquery.models.worker.DatasetRegistry;
+import com.bakdata.conquery.models.worker.Namespace;
 import com.bakdata.conquery.resources.api.MeResource;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -34,7 +35,7 @@ public class MeProcessor {
 	@Inject
 	private MetaStorage storage;
 	@Inject
-	private DatasetRegistry datasetRegistry;
+	private DatasetRegistry<? extends Namespace> datasetRegistry;
 
 	/**
 	 * Generates a summary of a user. It contains its name, the groups it belongs to and its permissions on a dataset.

diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java
@@ -60,7 +60,6 @@
 import com.bakdata.conquery.models.execution.ManagedExecution;
 import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
 import com.bakdata.conquery.models.identifiable.mapping.IdPrinter;
-import com.bakdata.conquery.models.messages.namespaces.specific.CancelQuery;
 import com.bakdata.conquery.models.query.ExecutionManager;
 import com.bakdata.conquery.models.query.ManagedQuery;
 import com.bakdata.conquery.models.query.SingleTableResult;
@@ -86,7 +85,7 @@
 public class QueryProcessor {
 
 	@Inject
-	private DatasetRegistry datasetRegistry;
+	private DatasetRegistry<? extends Namespace> datasetRegistry;
 	@Inject
 	private MetaStorage storage;
 	@Inject
@@ -305,11 +304,8 @@ public void cancel(Subject subject, Dataset dataset, ManagedExecution query) {
 
 		log.info("User[{}] cancelled Query[{}]", subject.getId(), query.getId());
 
-		final Namespace namespace = datasetRegistry.get(dataset.getId());
-
-		query.reset();
-
-		namespace.sendToAll(new CancelQuery(query.getId()));
+		final ExecutionManager executionManager = datasetRegistry.get(dataset.getId()).getExecutionManager();
+		executionManager.cancelQuery(dataset, query);
 	}
 
 	public void patchQuery(Subject subject, ManagedExecution execution, MetaDataPatch patch) {
@@ -417,7 +413,7 @@ public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uri
 
 
 		if (execution.getState() == ExecutionState.FAILED) {
-			throw ConqueryError.ContextError.fromErrorInfo(execution.getError());
+			throw new ConqueryError.ExecutionProcessingError();
 		}
 
 
@@ -465,7 +461,7 @@ public Stream<Map<String, String>> resolveEntities(Subject subject, List<FilterV
 		}
 
 		if (execution.getState() == ExecutionState.FAILED) {
-			throw ConqueryError.ContextError.fromErrorInfo(execution.getError());
+			throw new ConqueryError.ExecutionProcessingError();
 		}
 
 		final SingleTableResult result = (SingleTableResult) execution;