-
Notifications
You must be signed in to change notification settings - Fork 18
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* started * test and docs * Docs and sources * missed window ffunction * readme docs for unioning sources * docs * changelog * update package versions * joe feedback * readme tweaks following hubspot * update readme * swap source definition template * working * fix source relation * fix * bk * constant expression * constant expression errors * add source_relation to audit_log * docs * readme * joe feedback * source check * Apply suggestions from code review Co-authored-by: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> --------- Co-authored-by: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com>
- Loading branch information
1 parent
8b9ca18
commit 94c1b56
Showing
47 changed files
with
663 additions
and
118 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| {% macro union_zendesk_connections(connection_dictionary, single_source_name, single_table_name) %} | ||
|
|
||
| {{ adapter.dispatch('union_zendesk_connections', 'zendesk_source') (connection_dictionary, single_source_name, single_table_name) }} | ||
|
|
||
| {%- endmacro %} | ||
|
|
||
| {% macro default__union_zendesk_connections(connection_dictionary, single_source_name, single_table_name) %} | ||
|
|
||
| {% if connection_dictionary %} | ||
| {# For unioning #} | ||
| {%- set relations = [] -%} | ||
| {%- for connection in connection_dictionary -%} | ||
|
|
||
| {%- set relation=adapter.get_relation( | ||
| database=source(connection.name, single_table_name).database, | ||
| schema=source(connection.name, single_table_name).schema, | ||
| identifier=source(connection.name, single_table_name).identifier) if var('has_defined_sources', false) | ||
|
|
||
| else adapter.get_relation( | ||
| database=connection.database if connection.database else target.database, | ||
| schema=connection.schema if connection.schema else single_source_name, | ||
| identifier=single_table_name | ||
| ) | ||
| -%} | ||
|
|
||
| {%- if relation is not none -%} | ||
| {%- do relations.append(relation) -%} | ||
| {%- endif -%} | ||
|
|
||
| {%- endfor -%} | ||
|
|
||
| {%- if relations != [] -%} | ||
| {{ zendesk_source.zendesk_union_relations(relations) }} | ||
| {%- else -%} | ||
| {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%} | ||
| {{ exceptions.warn("\n\nPlease be aware: The " ~ single_source_name ~ "." ~ single_table_name ~ " table was not found in your schema(s). The Fivetran Data Model will create a completely empty staging model as to not break downstream transformations. To turn off these warnings, set the `fivetran__remove_empty_table_warnings` variable to TRUE (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details).\n") }} | ||
| {% endif -%} | ||
| select | ||
| cast(null as {{ dbt.type_string() }}) as _dbt_source_relation | ||
| limit 0 | ||
| {%- endif -%} | ||
|
|
||
| {% else %} | ||
| {# Not unioning #} | ||
| {%- set relation=adapter.get_relation( | ||
| database=source(single_source_name, single_table_name).database, | ||
| schema=source(single_source_name, single_table_name).schema, | ||
| identifier=source(single_source_name, single_table_name).identifier | ||
| ) -%} | ||
|
|
||
| {%- if relation is not none -%} | ||
| select | ||
| {{ dbt_utils.star(from=source(single_source_name, single_table_name)) }} | ||
| from {{ source(single_source_name, single_table_name) }} as source_table | ||
|
|
||
| {% else %} | ||
| {% if execute and not var('fivetran__remove_empty_table_warnings', false) -%} | ||
| {{ exceptions.warn("\n\nPlease be aware: The " ~ single_source_name|upper ~ "." ~ single_table_name|upper ~ " table was not found in your schema(s). The Fivetran Data Model will create a completely empty staging model as to not break downstream transformations. To turn off these warnings, set the `fivetran__remove_empty_table_warnings` variable to TRUE (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details).\n") }} | ||
| {% endif -%} | ||
|
|
||
| select | ||
| cast(null as {{ dbt.type_string() }}) as _dbt_source_relation | ||
| limit 0 | ||
| {%- endif -%} | ||
| {% endif -%} | ||
|
|
||
| {%- endmacro %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| {% macro apply_source_relation() -%} | ||
|
|
||
| {{ adapter.dispatch('apply_source_relation', 'zendesk_source') () }} | ||
|
|
||
| {%- endmacro %} | ||
|
|
||
| {% macro default__apply_source_relation() -%} | ||
|
|
||
| {% if var('zendesk_sources', []) != [] %} | ||
| , _dbt_source_relation as source_relation | ||
| {% else %} | ||
| , '{{ var("zendesk_database", target.database) }}' || '.'|| '{{ var("zendesk_schema", "zendesk") }}' as source_relation | ||
| {% endif %} | ||
|
|
||
| {%- endmacro %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| {# Adapted from dbt_utils.union_relations() #} | ||
|
|
||
| {%- macro zendesk_union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%} | ||
| {{ return(adapter.dispatch('zendesk_union_relations', 'zendesk_source')(relations, aliases, column_override, include, exclude, source_column_name, where)) }} | ||
| {% endmacro %} | ||
|
|
||
| {%- macro default__zendesk_union_relations(relations, aliases=none, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%} | ||
|
|
||
| {%- if exclude and include -%} | ||
| {{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }} | ||
| {%- endif -%} | ||
|
|
||
| {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} | ||
| {%- if not execute %} | ||
| {{ return('') }} | ||
| {% endif -%} | ||
|
|
||
| {%- set column_override = column_override if column_override is not none else {} -%} | ||
|
|
||
| {%- set relation_columns = {} -%} | ||
| {%- set column_superset = {} -%} | ||
| {%- set all_excludes = [] -%} | ||
| {%- set all_includes = [] -%} | ||
|
|
||
| {%- if exclude -%} | ||
| {%- for exc in exclude -%} | ||
| {%- do all_excludes.append(exc | lower) -%} | ||
| {%- endfor -%} | ||
| {%- endif -%} | ||
|
|
||
| {%- if include -%} | ||
| {%- for inc in include -%} | ||
| {%- do all_includes.append(inc | lower) -%} | ||
| {%- endfor -%} | ||
| {%- endif -%} | ||
|
|
||
| {%- for relation in relations -%} | ||
|
|
||
| {%- do relation_columns.update({relation: []}) -%} | ||
|
|
||
| {%- do dbt_utils._is_relation(relation, 'zendesk_union_relations') -%} | ||
| {%- do dbt_utils._is_ephemeral(relation, 'zendesk_union_relations') -%} | ||
| {%- set cols = adapter.get_columns_in_relation(relation) -%} | ||
| {%- for col in cols -%} | ||
|
|
||
| {#- If an exclude list was provided and the column is in the list, do nothing -#} | ||
| {%- if exclude and col.column | lower in all_excludes -%} | ||
|
|
||
| {#- If an include list was provided and the column is not in the list, do nothing -#} | ||
| {%- elif include and col.column | lower not in all_includes -%} | ||
|
|
||
| {#- Otherwise add the column to the column superset -#} | ||
| {%- else -%} | ||
|
|
||
| {#- update the list of columns in this relation -#} | ||
| {%- do relation_columns[relation].append(col.column) -%} | ||
|
|
||
| {%- if col.column in column_superset -%} | ||
|
|
||
| {%- set stored = column_superset[col.column] -%} | ||
| {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} | ||
|
|
||
| {%- do column_superset.update({col.column: col}) -%} | ||
|
|
||
| {%- endif %} | ||
|
|
||
| {%- else -%} | ||
|
|
||
| {%- do column_superset.update({col.column: col}) -%} | ||
|
|
||
| {%- endif -%} | ||
|
|
||
| {%- endif -%} | ||
|
|
||
| {%- endfor -%} | ||
| {%- endfor -%} | ||
|
|
||
| {%- set ordered_column_names = column_superset.keys() -%} | ||
| {%- set dbt_command = flags.WHICH -%} | ||
|
|
||
|
|
||
| {% if dbt_command in ['run', 'build'] %} | ||
| {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} | ||
| {%- set relations_string -%} | ||
| {%- for relation in relations -%} | ||
| {{ relation.name }} | ||
| {%- if not loop.last %}, {% endif -%} | ||
| {%- endfor -%} | ||
| {%- endset -%} | ||
|
|
||
| {%- set error_message -%} | ||
| There were no columns found to union for relations {{ relations_string }} | ||
| {%- endset -%} | ||
|
|
||
| {{ exceptions.raise_compiler_error(error_message) }} | ||
| {%- endif -%} | ||
| {%- endif -%} | ||
|
|
||
| {%- for relation in relations %} | ||
|
|
||
| ( | ||
| select | ||
|
|
||
| {%- if source_column_name is not none %} | ||
| cast({{ dbt.string_literal(relation.database ~ '.' ~ relation.schema) }} as {{ dbt.type_string() }}) as {{ source_column_name }}, | ||
| {%- endif %} | ||
|
|
||
| {% for col_name in ordered_column_names -%} | ||
|
|
||
| {%- set col = column_superset[col_name] %} | ||
| {%- set col_type = column_override.get(col.column, col.data_type) %} | ||
| {%- set col_name = adapter.quote(col_name) if col_name in relation_columns[relation] else 'null' %} | ||
| cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} {% if not loop.last %},{% endif -%} | ||
|
|
||
| {%- endfor %} | ||
|
|
||
| {# This alias is the only addition made to thr dbt_utils.union_relations() code. Avoids errors if the table is named a reserved keyword #} | ||
| from {{ aliases[loop.index0] if aliases else relation }} as unioned_relation_{{ loop.index }} | ||
|
|
||
| {% if where -%} | ||
| where {{ where }} | ||
| {%- endif %} | ||
| ) | ||
|
|
||
| {% if not loop.last -%} | ||
| union all | ||
| {% endif -%} | ||
|
|
||
| {%- endfor -%} | ||
|
|
||
| {%- endmacro -%} |
Oops, something went wrong.