+ * A ConQuery is a graph that has a {@link com.bakdata.conquery.apiv1.query.QueryDescription} as its root. + * The children of the root are of type {@link com.bakdata.conquery.apiv1.query.CQElement}. + * + * @param type of the node to convert + */ +public interface NodeConverter extends Converter { + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverterService.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverterService.java new file mode 100644 index 0000000000..f174a48265 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverterService.java @@ -0,0 +1,32 @@ +package com.bakdata.conquery.sql.conversion; + +import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; + +/** + * Entry point for converting {@link QueryDescription} to an SQL query. + */ +public class NodeConverterService extends ConverterService { + + private final SqlDialect dialect; + private final SqlConnectorConfig config; + + public NodeConverterService(SqlDialect dialect, SqlConnectorConfig config) { + super(dialect.getNodeConverters()); + this.dialect = dialect; + this.config = config; + } + + public ConversionContext convert(QueryDescription queryDescription) { + ConversionContext initialCtx = ConversionContext.builder() + .config(config) + .nodeConverterService(this) + .sqlDialect(this.dialect) + .build(); + return convert(queryDescription, initialCtx); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java new file mode 100644 index 0000000000..8ee38b2f56 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java @@ -0,0 +1,22 @@ +package com.bakdata.conquery.sql.conversion; + +import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.sql.SqlQuery; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; +import org.jooq.conf.ParamType; + +public class SqlConverter { + + private final NodeConverterService nodeConverterService; + + public SqlConverter(SqlDialect dialect, SqlConnectorConfig config) { + this.nodeConverterService = new NodeConverterService(dialect, config); + } + + public SqlQuery convert(QueryDescription queryDescription) { + ConversionContext converted = nodeConverterService.convert(queryDescription); + return new SqlQuery(converted.getFinalQuery().getSQL(ParamType.INLINED)); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java new file mode 100644 index 0000000000..b07383dacf --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java @@ -0,0 +1,44 @@ +package com.bakdata.conquery.sql.conversion.context; + +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.sql.conversion.NodeConverterService; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; +import lombok.Builder; +import lombok.Singular; +import lombok.Value; +import lombok.With; +import org.jooq.Record; +import org.jooq.Select; + +import java.util.List; + +@Value +@With +@Builder(toBuilder = true) +public class ConversionContext { + + SqlConnectorConfig config; + NodeConverterService nodeConverterService; + SqlDialect sqlDialect; + @Singular + List querySteps; + Select finalQuery; + boolean negation; + CDateRange dateRestrictionRange; + int queryStepCounter; + + + public boolean dateRestrictionActive() { + return this.dateRestrictionRange != null; + } + + public ConversionContext withQueryStep(QueryStep queryStep) { + return this.toBuilder() + .queryStep(queryStep) + .queryStepCounter(queryStepCounter + 1) + .build(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java new file mode 100644 index 0000000000..8e42dde561 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java @@ -0,0 +1,69 @@ +package com.bakdata.conquery.sql.conversion.context.selects; + +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Stream; + +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import lombok.Builder; +import lombok.Value; +import lombok.With; +import org.jooq.Field; + +/** + * {@link ConceptSelects} represent all select fields of a {@link CQConcept}. + */ +@Value +@With +@Builder(toBuilder = true) +public class ConceptSelects implements Selects { + + Field primaryColumn; + Optional> dateRestriction; + Optional> validityDate; + List> eventSelect; + List> eventFilter; + List> groupSelect; + List> groupFilter; + + @Override + public ConceptSelects byName(String qualifier) { + return builder() + .primaryColumn(this.mapFieldToQualifier(qualifier, this.primaryColumn)) + .dateRestriction(this.mapFieldStreamToQualifier(qualifier, this.dateRestriction.stream()).findFirst()) + .validityDate(this.mapFieldStreamToQualifier(qualifier, this.validityDate.stream()).findFirst()) + .eventSelect(this.mapFieldStreamToQualifier(qualifier, this.eventSelect.stream()).toList()) + .eventFilter(this.mapFieldStreamToQualifier(qualifier, this.eventFilter.stream()).toList()) + .groupSelect(this.mapFieldStreamToQualifier(qualifier, this.groupSelect.stream()).toList()) + .groupFilter(this.mapFieldStreamToQualifier(qualifier, this.groupFilter.stream()).toList()) + .build(); + } + + @Override + public List> all() { + return Stream.concat( + this.primaryColumnAndValidityDate(), + this.explicitSelects().stream() + ).toList(); + } + + private Stream> primaryColumnAndValidityDate() { + return Stream.concat( + Stream.of(this.primaryColumn), + this.validityDate.stream() + ); + } + + @Override + public List> explicitSelects() { + return Stream.of( + this.dateRestriction.stream(), + this.eventSelect.stream(), + this.eventFilter.stream(), + this.groupSelect.stream(), + this.groupFilter.stream() + ).flatMap(Function.identity()).toList(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java new file mode 100644 index 0000000000..1ce36e8ae8 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java @@ -0,0 +1,98 @@ +package com.bakdata.conquery.sql.conversion.context.selects; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Value; +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** + * {@link MergedSelects} represent the combination of multiple {@link Selects}. + * Default selects fields of multiple {@link Selects} will be merged and special select fields like the primary column + * or validity dates will be unified or aggregated due to defined policies. + */ +@Value +@AllArgsConstructor(access = AccessLevel.PRIVATE) +public class MergedSelects implements Selects { + + String PRIMARY_COLUMN_ALIAS = "primary_column"; + + Field primaryColumn; + + Optional> validityDate; + + /** + * A merged list of all select fields, except the primary column and validity date, + * of the {@link QueryStep}'s passed to the {@link MergedSelects} constructor. + * Each field name is qualified with its associated CTE name. + */ + List> mergedSelects; + + public MergedSelects(List querySteps) { + this.primaryColumn = this.coalescePrimaryColumns(querySteps); + this.validityDate = this.extractValidityDate(querySteps); + this.mergedSelects = this.mergeSelects(querySteps); + } + + private Field coalescePrimaryColumns(List querySteps) { + List> primaryColumns = querySteps.stream() + .map(queryStep -> this.mapFieldToQualifier(queryStep.getCteName(), queryStep.getSelects().getPrimaryColumn())) + .toList(); + return DSL.coalesce((Object) primaryColumns.get(0), primaryColumns.subList(1, primaryColumns.size()).toArray()) + .as(PRIMARY_COLUMN_ALIAS); + } + + private Optional> extractValidityDate(List querySteps) { + // TODO: date aggregation... + if (querySteps.isEmpty()) { + return Optional.empty(); + } + QueryStep firstQueryStep = querySteps.get(0); + return this.mapFieldStreamToQualifier(firstQueryStep.getCteName(), firstQueryStep.getSelects().getValidityDate().stream()) + .findFirst(); + } + + private List> mergeSelects(List queriesToJoin) { + return queriesToJoin.stream() + .flatMap(queryStep -> queryStep.getSelects().explicitSelects().stream() + .map(field -> this.mapFieldToQualifier(queryStep.getCteName(), field))) + .toList(); + } + + @Override + public MergedSelects byName(String qualifier) { + return new MergedSelects( + this.mapFieldToQualifier(qualifier, this.primaryColumn), + this.mapFieldStreamToQualifier(qualifier, this.validityDate.stream()).findFirst(), + this.mergedSelects.stream() + .map(field -> this.mapFieldToQualifier(qualifier, field)) + .toList() + ); + } + + @Override + public List> all() { + return Stream.concat( + this.primaryColumnAndValidityDate(), + this.mergedSelects.stream() + ).toList(); + } + + private Stream> primaryColumnAndValidityDate() { + return Stream.concat( + Stream.of(this.primaryColumn), + this.validityDate.stream() + ); + } + + @Override + public List> explicitSelects() { + return this.mergedSelects; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java new file mode 100644 index 0000000000..6229c247a9 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java @@ -0,0 +1,63 @@ +package com.bakdata.conquery.sql.conversion.context.selects; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import org.jooq.Field; +import org.jooq.impl.DSL; + +public interface Selects { + + Field getPrimaryColumn(); + + Optional> getValidityDate(); + + /** + * Returns the selected columns as fully qualified reference. + * + * @param qualifier the table name that creates these selects + * @return selects as fully qualified reference + * @see Selects#mapFieldToQualifier(String, Field) + */ + Selects byName(String qualifier); + + /** + * @return A list of all select fields including the primary column and validity date. + */ + List> all(); + + /** + * List of columns that the user explicitly referenced, either via a filter or a select. + * + * @return A list of all select fields WITHOUT implicitly selected columns like the primary column and validity date. + */ + List> explicitSelects(); + + default Stream> mapFieldStreamToQualifier(String qualifier, Stream> objectField) { + return objectField.map(column -> this.mapFieldToQualifier(qualifier, column)); + } + + /** + * Converts a select to its fully qualified reference. + * + * + * Example: + * {@code + * with a as (select c1 - c2 as c + * from t1) + * select t1.c + * from a + * } + * + * This function maps the select {@code c1 - c2 as c} to {@code t1.c}. + * + * @param qualifier + * @param field + * @return + */ + default Field mapFieldToQualifier(String qualifier, Field field) { + return DSL.field(DSL.name(qualifier, field.getName())); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/LogicalOperation.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/LogicalOperation.java new file mode 100644 index 0000000000..9f0351c854 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/LogicalOperation.java @@ -0,0 +1,6 @@ +package com.bakdata.conquery.sql.conversion.context.step; + +public enum LogicalOperation { + AND, + OR +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java new file mode 100644 index 0000000000..9d72ec56bd --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java @@ -0,0 +1,40 @@ +package com.bakdata.conquery.sql.conversion.context.step; + +import java.util.List; + +import com.bakdata.conquery.sql.conversion.context.selects.Selects; +import lombok.Builder; +import lombok.Value; +import org.jooq.Condition; +import org.jooq.Record; +import org.jooq.TableLike; +import org.jooq.impl.DSL; + +/** + * Intermediate representation of an SQL query. + */ +@Value +@Builder +public class QueryStep { + + String cteName; + Selects selects; + TableLike fromTable; + List conditions; + /** + * The CTEs referenced by this QueryStep + */ + List predecessors; + + public static TableLike toTableLike(String fromTableName) { + return DSL.table(DSL.name(fromTableName)); + } + + /** + * @return All selects re-mapped to a qualifier, which is the cteName of this QueryStep. + */ + public Selects getQualifiedSelects() { + return this.selects.byName(this.cteName); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStepTransformer.java new file mode 100644 index 0000000000..e41c457235 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStepTransformer.java @@ -0,0 +1,59 @@ +package com.bakdata.conquery.sql.conversion.context.step; + +import java.util.List; +import java.util.stream.Stream; + +import org.jooq.CommonTableExpression; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Select; +import org.jooq.impl.DSL; + +/** + * Transformer for translating the intermediate representation of {@link QueryStep} into the final SQL query. + */ +public class QueryStepTransformer { + + private final DSLContext dslContext; + + public QueryStepTransformer(DSLContext dslContext) { + this.dslContext = dslContext; + } + + /** + * Converts a given {@link QueryStep} into an executable SELECT statement. + */ + public Select toSelectQuery(QueryStep queryStep) { + return this.dslContext.with(this.constructPredecessorCteList(queryStep)) + .select(queryStep.getSelects().all()) + .from(queryStep.getFromTable()) + .where(queryStep.getConditions()); + } + + private List> constructPredecessorCteList(QueryStep queryStep) { + return queryStep.getPredecessors().stream() + .flatMap(predecessor -> this.toCteList(predecessor).stream()) + .toList(); + } + + private List> toCteList(QueryStep queryStep) { + return Stream.concat( + this.predecessorCtes(queryStep), + Stream.of(this.toCte(queryStep)) + ).toList(); + } + + private Stream> predecessorCtes(QueryStep queryStep) { + return queryStep.getPredecessors().stream() + .flatMap(predecessor -> this.toCteList(predecessor).stream()); + } + + private CommonTableExpression toCte(QueryStep queryStep) { + return DSL.name(queryStep.getCteName()).as( + this.dslContext.select(queryStep.getSelects().all()) + .from(queryStep.getFromTable()) + .where(queryStep.getConditions()) + ); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/StepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/StepJoiner.java new file mode 100644 index 0000000000..84b103f955 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/StepJoiner.java @@ -0,0 +1,89 @@ +package com.bakdata.conquery.sql.conversion.context.step; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.selects.MergedSelects; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableLike; +import org.jooq.TableOnConditionStep; +import org.jooq.impl.DSL; + +public class StepJoiner { + + public static ConversionContext joinChildren(Iterable children, ConversionContext context, LogicalOperation logicalOperation) { + + ConversionContext childrenContext = context; + for (CQElement childNode : children) { + childrenContext = context.getNodeConverterService().convert(childNode, childrenContext); + } + + List queriesToJoin = childrenContext.getQuerySteps(); + QueryStep andQueryStep = QueryStep.builder() + .cteName(constructJoinedQueryStepLabel(queriesToJoin, logicalOperation)) + .selects(new MergedSelects(queriesToJoin)) + .fromTable(constructJoinedTable(queriesToJoin, logicalOperation, context)) + .conditions(Collections.emptyList()) + .predecessors(queriesToJoin) + .build(); + + return context.withQuerySteps(List.of(andQueryStep)); + } + + private static String constructJoinedQueryStepLabel(List queriesToJoin, LogicalOperation logicalOperation) { + + String labelConnector = switch (logicalOperation) { + case AND -> "_AND_"; + case OR -> "_OR_"; + }; + + return queriesToJoin.stream() + .map(QueryStep::getCteName) + .collect(Collectors.joining(labelConnector)); + } + + private static TableLike constructJoinedTable(List queriesToJoin, LogicalOperation logicalOperation, ConversionContext context) { + + Table joinedQuery = getIntitialJoinTable(queriesToJoin); + + SqlFunctionProvider functionProvider = context.getSqlDialect().getFunction(); + JoinType joinType = switch (logicalOperation) { + case AND -> functionProvider::innerJoin; + case OR -> functionProvider::fullOuterJoin; + }; + + for (int i = 0; i < queriesToJoin.size() - 1; i++) { + + QueryStep leftPartQS = queriesToJoin.get(i); + QueryStep rightPartQS = queriesToJoin.get(i + 1); + + Field leftPartPrimaryColumn = leftPartQS.getQualifiedSelects().getPrimaryColumn(); + Field rightPartPrimaryColumn = rightPartQS.getQualifiedSelects().getPrimaryColumn(); + + joinedQuery = joinType.join(joinedQuery, rightPartQS, leftPartPrimaryColumn, rightPartPrimaryColumn); + } + + return joinedQuery; + } + + private static Table getIntitialJoinTable(List queriesToJoin) { + return DSL.table(DSL.name(queriesToJoin.get(0).getCteName())); + } + + @FunctionalInterface + private interface JoinType { + TableOnConditionStep join( + Table leftPartQueryBase, + QueryStep rightPartQS, + Field leftPartPrimaryColumn, + Field rightPartPrimaryColumn + ); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java new file mode 100644 index 0000000000..31404da21d --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java @@ -0,0 +1,24 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.step.LogicalOperation; +import com.bakdata.conquery.sql.conversion.context.step.StepJoiner; + +public class CQAndConverter implements NodeConverter { + + @Override + public Class getConversionClass() { + return CQAnd.class; + } + + @Override + public ConversionContext convert(CQAnd andNode, ConversionContext context) { + if (andNode.getChildren().size() == 1) { + return context.getNodeConverterService().convert(andNode.getChildren().get(0), context); + } + return StepJoiner.joinChildren(andNode.getChildren(), context, LogicalOperation.AND); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java new file mode 100644 index 0000000000..db4e9b43b2 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java @@ -0,0 +1,208 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.conversion.filter.FilterConverterService; +import com.bakdata.conquery.sql.conversion.select.SelectConverterService; +import org.jooq.Condition; +import org.jooq.Field; + +public class CQConceptConverter implements NodeConverter { + + private final FilterConverterService filterConverterService; + private final SelectConverterService selectConverterService; + + public CQConceptConverter(FilterConverterService filterConverterService, SelectConverterService selectConverterService) { + this.filterConverterService = filterConverterService; + this.selectConverterService = selectConverterService; + } + + @Override + public Class getConversionClass() { + return CQConcept.class; + } + + @Override + public ConversionContext convert(CQConcept node, ConversionContext context) { + + if (node.getTables().size() > 1) { + throw new UnsupportedOperationException("Can't handle concepts with multiple tables for now."); + } + + ConceptPreprocessingService preprocessingService = new ConceptPreprocessingService(node, context); + CQTable table = node.getTables().get(0); + String conceptLabel = this.getConceptLabel(node, context); + + QueryStep preprocessingStep = preprocessingService.buildPreprocessingQueryStepForTable(conceptLabel, table); + QueryStep dateRestriction = this.buildDateRestrictionQueryStep(context, node, conceptLabel, preprocessingStep); + QueryStep eventSelect = this.buildEventSelectQueryStep(context, table, conceptLabel, dateRestriction); + QueryStep eventFilter = this.buildEventFilterQueryStep(context, table, conceptLabel, eventSelect); + QueryStep finalStep = this.buildFinalQueryStep(conceptLabel, eventFilter); + + return context.withQueryStep(finalStep); + } + + private String getConceptLabel(CQConcept node, ConversionContext context) { + // only relevant for debugging purposes as it will be part of the generated SQL query + // we prefix each cte name of a concept with an incrementing counter to prevent naming collisions if the same concept is selected multiple times + return "%s_%s".formatted( + context.getQueryStepCounter(), + node.getUserOrDefaultLabel(Locale.ENGLISH) + .toLowerCase() + .replace(' ', '_') + .replaceAll("\\s", "_") + ); + } + + /** + * selects: + * - all of previous step + */ + private QueryStep buildDateRestrictionQueryStep( + ConversionContext context, + CQConcept node, + String conceptLabel, + QueryStep previous + ) { + if (((ConceptSelects) previous.getSelects()).getDateRestriction().isEmpty()) { + return previous; + } + + ConceptSelects dateRestrictionSelects = this.prepareDateRestrictionSelects(node, previous); + List dateRestriction = this.buildDateRestriction(context, previous); + + return QueryStep.builder() + .cteName(createCteName(conceptLabel, "_date_restriction")) + .fromTable(QueryStep.toTableLike(previous.getCteName())) + .selects(dateRestrictionSelects) + .conditions(dateRestriction) + .predecessors(List.of(previous)) + .build(); + } + + /** + * selects: + * - all of previous steps + * - transformed columns with selects + */ + private QueryStep buildEventSelectQueryStep( + ConversionContext context, + CQTable table, + String conceptLabel, QueryStep previous + ) { + if (table.getSelects().isEmpty()) { + return previous; + } + + ConceptSelects eventSelectSelects = this.prepareEventSelectSelects(context, table, previous); + + return QueryStep.builder() + .cteName(createCteName(conceptLabel, "_event_select")) + .fromTable(QueryStep.toTableLike(previous.getCteName())) + .selects(eventSelectSelects) + .conditions(Collections.emptyList()) + .predecessors(List.of(previous)) + .build(); + } + + /** + * selects: + * - all of previous step + * - remove filter + */ + private QueryStep buildEventFilterQueryStep( + ConversionContext context, + CQTable table, + String conceptLabel, + QueryStep previous + ) { + if (table.getFilters().isEmpty()) { + return previous; + } + + ConceptSelects eventFilterSelects = this.prepareEventFilterSelects(previous); + List eventFilterConditions = this.buildEventFilterConditions(context, table); + + return QueryStep.builder() + .cteName(createCteName(conceptLabel, "_event_filter")) + .fromTable(QueryStep.toTableLike(previous.getCteName())) + .selects(eventFilterSelects) + .conditions(eventFilterConditions) + .predecessors(List.of(previous)) + .build(); + } + + private ConceptSelects prepareDateRestrictionSelects(CQConcept node, QueryStep previous) { + ConceptSelects.ConceptSelectsBuilder selectsBuilder = ((ConceptSelects) previous.getQualifiedSelects()).toBuilder(); + selectsBuilder.dateRestriction(Optional.empty()); + if (node.isExcludeFromTimeAggregation()) { + selectsBuilder.validityDate(Optional.empty()); + } + return selectsBuilder.build(); + } + + private List buildDateRestriction(ConversionContext context, QueryStep previous) { + return ((ConceptSelects) previous.getSelects()).getDateRestriction() + .map(dateRestrictionColumn -> getDateRestrictionAsCondition(context, previous, dateRestrictionColumn)) + .orElseGet(Collections::emptyList); + } + + private static List getDateRestrictionAsCondition(ConversionContext context, QueryStep previous, Field dateRestrictionColumn) { + return previous.getSelects().getValidityDate().stream() + .map(validityDateColumn -> context.getSqlDialect().getFunction().dateRestriction(dateRestrictionColumn, validityDateColumn)) + .toList(); + } + + private ConceptSelects prepareEventSelectSelects( + ConversionContext context, + CQTable table, + QueryStep previous + ) { + return ((ConceptSelects) previous.getQualifiedSelects()).withEventSelect(this.getEventSelects(context, table)); + } + + private ConceptSelects prepareEventFilterSelects(QueryStep previous) { + return ((ConceptSelects) previous.getQualifiedSelects()).withEventFilter(Collections.emptyList()); + } + + private List buildEventFilterConditions(ConversionContext context, CQTable table) { + return table.getFilters().stream() + .map(filterValue -> this.filterConverterService.convert(filterValue, context)) + .toList(); + } + + private List> getEventSelects(ConversionContext context, CQTable table) { + return table.getSelects().stream() + .map(select -> (Field) this.selectConverterService.convert(select, context)) + .toList(); + } + + /** + * selects: + * - all of previous step + */ + private QueryStep buildFinalQueryStep(String conceptLabel, QueryStep previous) { + ConceptSelects finalSelects = ((ConceptSelects) previous.getQualifiedSelects()); + return QueryStep.builder() + .cteName(createCteName(conceptLabel, "")) + .fromTable(QueryStep.toTableLike(previous.getCteName())) + .selects(finalSelects) + .conditions(Collections.emptyList()) + .predecessors(List.of(previous)) + .build(); + } + + private static String createCteName(String conceptLabel, String suffix) { + return "concept_%s%s".formatted(conceptLabel, suffix); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java new file mode 100644 index 0000000000..37e3c1d56f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java @@ -0,0 +1,20 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; + +public class CQDateRestrictionConverter implements NodeConverter { + + @Override + public ConversionContext convert(CQDateRestriction node, ConversionContext context) { + ConversionContext childContext = context.withDateRestrictionRange(CDateRange.of(node.getDateRange())); + return context.getNodeConverterService().convert(node.getChild(), childContext).withDateRestrictionRange(null); + } + + @Override + public Class getConversionClass() { + return CQDateRestriction.class; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java new file mode 100644 index 0000000000..ce1db05bd7 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; + +public class CQNegationConverter implements NodeConverter { + + @Override + public Class getConversionClass() { + return CQNegation.class; + } + + @Override + public ConversionContext convert(CQNegation negationNode, ConversionContext context) { + return this.convertChildWithNegationActive(negationNode.getChild(), context); + } + + private ConversionContext convertChildWithNegationActive(CQElement child, ConversionContext context) { + // TODO: handle negation properly after GroupSelect/GroupFilter has been implemented + // - anti-join vs. negating conditions + // - handle double negation + return context.getNodeConverterService() + .convert(child, context.withNegation(true)) + .withNegation(false); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java new file mode 100644 index 0000000000..7df18dedaf --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java @@ -0,0 +1,24 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.step.LogicalOperation; +import com.bakdata.conquery.sql.conversion.context.step.StepJoiner; + +public class CQOrConverter implements NodeConverter { + + @Override + public Class getConversionClass() { + return CQOr.class; + } + + @Override + public ConversionContext convert(CQOr orNode, ConversionContext context) { + if (orNode.getChildren().size() == 1) { + return context.getNodeConverterService().convert(orNode.getChildren().get(0), context); + } + return StepJoiner.joinChildren(orNode.getChildren(), context, LogicalOperation.OR); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java new file mode 100644 index 0000000000..2800d8b30a --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java @@ -0,0 +1,136 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import org.jooq.Field; +import org.jooq.impl.DSL; + +public class ConceptPreprocessingService { + + private static final String DATE_RESTRICTION_COLUMN_NAME = "date_restriction"; + private static final String VALIDITY_DATE_COLUMN_NAME_SUFFIX = "_validity_date"; + private final CQConcept concept; + private final ConversionContext context; + private final SqlFunctionProvider sqlFunctionProvider; + + public ConceptPreprocessingService(CQConcept concept, ConversionContext context) { + this.concept = concept; + this.context = context; + this.sqlFunctionProvider = this.context.getSqlDialect().getFunction(); + } + + /** + * selects: + * - (primary column) + * - date restriction + * - validity date + * - any filter (group/event) + * - any select (group/event) + */ + public QueryStep buildPreprocessingQueryStepForTable(String conceptLabel, CQTable table) { + + ConceptSelects.ConceptSelectsBuilder selectsBuilder = ConceptSelects.builder(); + + selectsBuilder.primaryColumn(DSL.field(context.getConfig().getPrimaryColumn())); + selectsBuilder.dateRestriction(this.getDateRestrictionSelect(table)); + selectsBuilder.validityDate(this.getValidityDateSelect(table, conceptLabel)); + + List
+ *
{@code + * with a as (select c1 - c2 as c + * from t1) + * select t1.c + * from a + * }
+ * This function maps the select {@code c1 - c2 as c} to {@code t1.c}. + * + * @param qualifier + * @param field + * @return + */ + default Field mapFieldToQualifier(String qualifier, Field field) { + return DSL.field(DSL.name(qualifier, field.getName())); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/LogicalOperation.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/LogicalOperation.java new file mode 100644 index 0000000000..9f0351c854 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/LogicalOperation.java @@ -0,0 +1,6 @@ +package com.bakdata.conquery.sql.conversion.context.step; + +public enum LogicalOperation { + AND, + OR +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java new file mode 100644 index 0000000000..9d72ec56bd --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java @@ -0,0 +1,40 @@ +package com.bakdata.conquery.sql.conversion.context.step; + +import java.util.List; + +import com.bakdata.conquery.sql.conversion.context.selects.Selects; +import lombok.Builder; +import lombok.Value; +import org.jooq.Condition; +import org.jooq.Record; +import org.jooq.TableLike; +import org.jooq.impl.DSL; + +/** + * Intermediate representation of an SQL query. + */ +@Value +@Builder +public class QueryStep { + + String cteName; + Selects selects; + TableLike fromTable; + List conditions; + /** + * The CTEs referenced by this QueryStep + */ + List predecessors; + + public static TableLike toTableLike(String fromTableName) { + return DSL.table(DSL.name(fromTableName)); + } + + /** + * @return All selects re-mapped to a qualifier, which is the cteName of this QueryStep. + */ + public Selects getQualifiedSelects() { + return this.selects.byName(this.cteName); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStepTransformer.java new file mode 100644 index 0000000000..e41c457235 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStepTransformer.java @@ -0,0 +1,59 @@ +package com.bakdata.conquery.sql.conversion.context.step; + +import java.util.List; +import java.util.stream.Stream; + +import org.jooq.CommonTableExpression; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Select; +import org.jooq.impl.DSL; + +/** + * Transformer for translating the intermediate representation of {@link QueryStep} into the final SQL query. + */ +public class QueryStepTransformer { + + private final DSLContext dslContext; + + public QueryStepTransformer(DSLContext dslContext) { + this.dslContext = dslContext; + } + + /** + * Converts a given {@link QueryStep} into an executable SELECT statement. + */ + public Select toSelectQuery(QueryStep queryStep) { + return this.dslContext.with(this.constructPredecessorCteList(queryStep)) + .select(queryStep.getSelects().all()) + .from(queryStep.getFromTable()) + .where(queryStep.getConditions()); + } + + private List> constructPredecessorCteList(QueryStep queryStep) { + return queryStep.getPredecessors().stream() + .flatMap(predecessor -> this.toCteList(predecessor).stream()) + .toList(); + } + + private List> toCteList(QueryStep queryStep) { + return Stream.concat( + this.predecessorCtes(queryStep), + Stream.of(this.toCte(queryStep)) + ).toList(); + } + + private Stream> predecessorCtes(QueryStep queryStep) { + return queryStep.getPredecessors().stream() + .flatMap(predecessor -> this.toCteList(predecessor).stream()); + } + + private CommonTableExpression toCte(QueryStep queryStep) { + return DSL.name(queryStep.getCteName()).as( + this.dslContext.select(queryStep.getSelects().all()) + .from(queryStep.getFromTable()) + .where(queryStep.getConditions()) + ); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/StepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/StepJoiner.java new file mode 100644 index 0000000000..84b103f955 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/StepJoiner.java @@ -0,0 +1,89 @@ +package com.bakdata.conquery.sql.conversion.context.step; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.selects.MergedSelects; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableLike; +import org.jooq.TableOnConditionStep; +import org.jooq.impl.DSL; + +public class StepJoiner { + + public static ConversionContext joinChildren(Iterable children, ConversionContext context, LogicalOperation logicalOperation) { + + ConversionContext childrenContext = context; + for (CQElement childNode : children) { + childrenContext = context.getNodeConverterService().convert(childNode, childrenContext); + } + + List queriesToJoin = childrenContext.getQuerySteps(); + QueryStep andQueryStep = QueryStep.builder() + .cteName(constructJoinedQueryStepLabel(queriesToJoin, logicalOperation)) + .selects(new MergedSelects(queriesToJoin)) + .fromTable(constructJoinedTable(queriesToJoin, logicalOperation, context)) + .conditions(Collections.emptyList()) + .predecessors(queriesToJoin) + .build(); + + return context.withQuerySteps(List.of(andQueryStep)); + } + + private static String constructJoinedQueryStepLabel(List queriesToJoin, LogicalOperation logicalOperation) { + + String labelConnector = switch (logicalOperation) { + case AND -> "_AND_"; + case OR -> "_OR_"; + }; + + return queriesToJoin.stream() + .map(QueryStep::getCteName) + .collect(Collectors.joining(labelConnector)); + } + + private static TableLike constructJoinedTable(List queriesToJoin, LogicalOperation logicalOperation, ConversionContext context) { + + Table joinedQuery = getIntitialJoinTable(queriesToJoin); + + SqlFunctionProvider functionProvider = context.getSqlDialect().getFunction(); + JoinType joinType = switch (logicalOperation) { + case AND -> functionProvider::innerJoin; + case OR -> functionProvider::fullOuterJoin; + }; + + for (int i = 0; i < queriesToJoin.size() - 1; i++) { + + QueryStep leftPartQS = queriesToJoin.get(i); + QueryStep rightPartQS = queriesToJoin.get(i + 1); + + Field leftPartPrimaryColumn = leftPartQS.getQualifiedSelects().getPrimaryColumn(); + Field rightPartPrimaryColumn = rightPartQS.getQualifiedSelects().getPrimaryColumn(); + + joinedQuery = joinType.join(joinedQuery, rightPartQS, leftPartPrimaryColumn, rightPartPrimaryColumn); + } + + return joinedQuery; + } + + private static Table getIntitialJoinTable(List queriesToJoin) { + return DSL.table(DSL.name(queriesToJoin.get(0).getCteName())); + } + + @FunctionalInterface + private interface JoinType { + TableOnConditionStep join( + Table leftPartQueryBase, + QueryStep rightPartQS, + Field leftPartPrimaryColumn, + Field rightPartPrimaryColumn + ); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java new file mode 100644 index 0000000000..31404da21d --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java @@ -0,0 +1,24 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.step.LogicalOperation; +import com.bakdata.conquery.sql.conversion.context.step.StepJoiner; + +public class CQAndConverter implements NodeConverter { + + @Override + public Class getConversionClass() { + return CQAnd.class; + } + + @Override + public ConversionContext convert(CQAnd andNode, ConversionContext context) { + if (andNode.getChildren().size() == 1) { + return context.getNodeConverterService().convert(andNode.getChildren().get(0), context); + } + return StepJoiner.joinChildren(andNode.getChildren(), context, LogicalOperation.AND); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java new file mode 100644 index 0000000000..db4e9b43b2 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java @@ -0,0 +1,208 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.conversion.filter.FilterConverterService; +import com.bakdata.conquery.sql.conversion.select.SelectConverterService; +import org.jooq.Condition; +import org.jooq.Field; + +public class CQConceptConverter implements NodeConverter { + + private final FilterConverterService filterConverterService; + private final SelectConverterService selectConverterService; + + public CQConceptConverter(FilterConverterService filterConverterService, SelectConverterService selectConverterService) { + this.filterConverterService = filterConverterService; + this.selectConverterService = selectConverterService; + } + + @Override + public Class getConversionClass() { + return CQConcept.class; + } + + @Override + public ConversionContext convert(CQConcept node, ConversionContext context) { + + if (node.getTables().size() > 1) { + throw new UnsupportedOperationException("Can't handle concepts with multiple tables for now."); + } + + ConceptPreprocessingService preprocessingService = new ConceptPreprocessingService(node, context); + CQTable table = node.getTables().get(0); + String conceptLabel = this.getConceptLabel(node, context); + + QueryStep preprocessingStep = preprocessingService.buildPreprocessingQueryStepForTable(conceptLabel, table); + QueryStep dateRestriction = this.buildDateRestrictionQueryStep(context, node, conceptLabel, preprocessingStep); + QueryStep eventSelect = this.buildEventSelectQueryStep(context, table, conceptLabel, dateRestriction); + QueryStep eventFilter = this.buildEventFilterQueryStep(context, table, conceptLabel, eventSelect); + QueryStep finalStep = this.buildFinalQueryStep(conceptLabel, eventFilter); + + return context.withQueryStep(finalStep); + } + + private String getConceptLabel(CQConcept node, ConversionContext context) { + // only relevant for debugging purposes as it will be part of the generated SQL query + // we prefix each cte name of a concept with an incrementing counter to prevent naming collisions if the same concept is selected multiple times + return "%s_%s".formatted( + context.getQueryStepCounter(), + node.getUserOrDefaultLabel(Locale.ENGLISH) + .toLowerCase() + .replace(' ', '_') + .replaceAll("\\s", "_") + ); + } + + /** + * selects: + * - all of previous step + */ + private QueryStep buildDateRestrictionQueryStep( + ConversionContext context, + CQConcept node, + String conceptLabel, + QueryStep previous + ) { + if (((ConceptSelects) previous.getSelects()).getDateRestriction().isEmpty()) { + return previous; + } + + ConceptSelects dateRestrictionSelects = this.prepareDateRestrictionSelects(node, previous); + List dateRestriction = this.buildDateRestriction(context, previous); + + return QueryStep.builder() + .cteName(createCteName(conceptLabel, "_date_restriction")) + .fromTable(QueryStep.toTableLike(previous.getCteName())) + .selects(dateRestrictionSelects) + .conditions(dateRestriction) + .predecessors(List.of(previous)) + .build(); + } + + /** + * selects: + * - all of previous steps + * - transformed columns with selects + */ + private QueryStep buildEventSelectQueryStep( + ConversionContext context, + CQTable table, + String conceptLabel, QueryStep previous + ) { + if (table.getSelects().isEmpty()) { + return previous; + } + + ConceptSelects eventSelectSelects = this.prepareEventSelectSelects(context, table, previous); + + return QueryStep.builder() + .cteName(createCteName(conceptLabel, "_event_select")) + .fromTable(QueryStep.toTableLike(previous.getCteName())) + .selects(eventSelectSelects) + .conditions(Collections.emptyList()) + .predecessors(List.of(previous)) + .build(); + } + + /** + * selects: + * - all of previous step + * - remove filter + */ + private QueryStep buildEventFilterQueryStep( + ConversionContext context, + CQTable table, + String conceptLabel, + QueryStep previous + ) { + if (table.getFilters().isEmpty()) { + return previous; + } + + ConceptSelects eventFilterSelects = this.prepareEventFilterSelects(previous); + List eventFilterConditions = this.buildEventFilterConditions(context, table); + + return QueryStep.builder() + .cteName(createCteName(conceptLabel, "_event_filter")) + .fromTable(QueryStep.toTableLike(previous.getCteName())) + .selects(eventFilterSelects) + .conditions(eventFilterConditions) + .predecessors(List.of(previous)) + .build(); + } + + private ConceptSelects prepareDateRestrictionSelects(CQConcept node, QueryStep previous) { + ConceptSelects.ConceptSelectsBuilder selectsBuilder = ((ConceptSelects) previous.getQualifiedSelects()).toBuilder(); + selectsBuilder.dateRestriction(Optional.empty()); + if (node.isExcludeFromTimeAggregation()) { + selectsBuilder.validityDate(Optional.empty()); + } + return selectsBuilder.build(); + } + + private List buildDateRestriction(ConversionContext context, QueryStep previous) { + return ((ConceptSelects) previous.getSelects()).getDateRestriction() + .map(dateRestrictionColumn -> getDateRestrictionAsCondition(context, previous, dateRestrictionColumn)) + .orElseGet(Collections::emptyList); + } + + private static List getDateRestrictionAsCondition(ConversionContext context, QueryStep previous, Field dateRestrictionColumn) { + return previous.getSelects().getValidityDate().stream() + .map(validityDateColumn -> context.getSqlDialect().getFunction().dateRestriction(dateRestrictionColumn, validityDateColumn)) + .toList(); + } + + private ConceptSelects prepareEventSelectSelects( + ConversionContext context, + CQTable table, + QueryStep previous + ) { + return ((ConceptSelects) previous.getQualifiedSelects()).withEventSelect(this.getEventSelects(context, table)); + } + + private ConceptSelects prepareEventFilterSelects(QueryStep previous) { + return ((ConceptSelects) previous.getQualifiedSelects()).withEventFilter(Collections.emptyList()); + } + + private List buildEventFilterConditions(ConversionContext context, CQTable table) { + return table.getFilters().stream() + .map(filterValue -> this.filterConverterService.convert(filterValue, context)) + .toList(); + } + + private List> getEventSelects(ConversionContext context, CQTable table) { + return table.getSelects().stream() + .map(select -> (Field) this.selectConverterService.convert(select, context)) + .toList(); + } + + /** + * selects: + * - all of previous step + */ + private QueryStep buildFinalQueryStep(String conceptLabel, QueryStep previous) { + ConceptSelects finalSelects = ((ConceptSelects) previous.getQualifiedSelects()); + return QueryStep.builder() + .cteName(createCteName(conceptLabel, "")) + .fromTable(QueryStep.toTableLike(previous.getCteName())) + .selects(finalSelects) + .conditions(Collections.emptyList()) + .predecessors(List.of(previous)) + .build(); + } + + private static String createCteName(String conceptLabel, String suffix) { + return "concept_%s%s".formatted(conceptLabel, suffix); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java new file mode 100644 index 0000000000..37e3c1d56f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java @@ -0,0 +1,20 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; + +public class CQDateRestrictionConverter implements NodeConverter { + + @Override + public ConversionContext convert(CQDateRestriction node, ConversionContext context) { + ConversionContext childContext = context.withDateRestrictionRange(CDateRange.of(node.getDateRange())); + return context.getNodeConverterService().convert(node.getChild(), childContext).withDateRestrictionRange(null); + } + + @Override + public Class getConversionClass() { + return CQDateRestriction.class; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java new file mode 100644 index 0000000000..ce1db05bd7 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; + +public class CQNegationConverter implements NodeConverter { + + @Override + public Class getConversionClass() { + return CQNegation.class; + } + + @Override + public ConversionContext convert(CQNegation negationNode, ConversionContext context) { + return this.convertChildWithNegationActive(negationNode.getChild(), context); + } + + private ConversionContext convertChildWithNegationActive(CQElement child, ConversionContext context) { + // TODO: handle negation properly after GroupSelect/GroupFilter has been implemented + // - anti-join vs. negating conditions + // - handle double negation + return context.getNodeConverterService() + .convert(child, context.withNegation(true)) + .withNegation(false); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java new file mode 100644 index 0000000000..7df18dedaf --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java @@ -0,0 +1,24 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.step.LogicalOperation; +import com.bakdata.conquery.sql.conversion.context.step.StepJoiner; + +public class CQOrConverter implements NodeConverter { + + @Override + public Class getConversionClass() { + return CQOr.class; + } + + @Override + public ConversionContext convert(CQOr orNode, ConversionContext context) { + if (orNode.getChildren().size() == 1) { + return context.getNodeConverterService().convert(orNode.getChildren().get(0), context); + } + return StepJoiner.joinChildren(orNode.getChildren(), context, LogicalOperation.OR); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java new file mode 100644 index 0000000000..2800d8b30a --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java @@ -0,0 +1,136 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import org.jooq.Field; +import org.jooq.impl.DSL; + +public class ConceptPreprocessingService { + + private static final String DATE_RESTRICTION_COLUMN_NAME = "date_restriction"; + private static final String VALIDITY_DATE_COLUMN_NAME_SUFFIX = "_validity_date"; + private final CQConcept concept; + private final ConversionContext context; + private final SqlFunctionProvider sqlFunctionProvider; + + public ConceptPreprocessingService(CQConcept concept, ConversionContext context) { + this.concept = concept; + this.context = context; + this.sqlFunctionProvider = this.context.getSqlDialect().getFunction(); + } + + /** + * selects: + * - (primary column) + * - date restriction + * - validity date + * - any filter (group/event) + * - any select (group/event) + */ + public QueryStep buildPreprocessingQueryStepForTable(String conceptLabel, CQTable table) { + + ConceptSelects.ConceptSelectsBuilder selectsBuilder = ConceptSelects.builder(); + + selectsBuilder.primaryColumn(DSL.field(context.getConfig().getPrimaryColumn())); + selectsBuilder.dateRestriction(this.getDateRestrictionSelect(table)); + selectsBuilder.validityDate(this.getValidityDateSelect(table, conceptLabel)); + + List