Filter out deleted accounts

CHANGELOG.md

@@ -1,3 +1,23 @@
+# dbt_quickbooks_source v0.11.0
+[PR #57](https://github.com/fivetran/dbt_quickbooks_source/pull/57) introduces the following updates:
+
+## Breaking Changes
+- We are currently bringing in records that have previously been deleted, which could be impacting reporting accuracy. Thus, we introduced the `_fivetran_deleted` field to filter out these records from the following staging models:
+  - `stg_quickbooks__account`
+  - `stg_quickbooks__bundle`
+  - `stg_quickbooks__customer`
+  - `stg_quickbooks__department`
+  - `stg_quickbooks__item`
+  - `stg_quickbooks__vendor`
+- As this will filter out deleted records that were previously being counted and we are adding a new field to these staging models, this will be a breaking change here and in `dbt_quickbooks`. 
+
+## Documentation Updates
+- Added the `_fivetran_deleted` field to `src_quickbooks` and `stg_quickbook` yml documentation.
+- Added the `_fivetran_deleted` field to the above corresponding seed files in integration tests.
+
+## Contibutors
+- [@brandonrf94](https://github.com/brandonrf94) [#56](https://github.com/fivetran/dbt_quickbooks_source/pull/56)
+
 # dbt_quickbooks_source v0.10.1
 [PR #53](https://github.com/fivetran/dbt_quickbooks_source/pull/53) introduces the following updates: 
 

README.md

@@ -52,7 +52,7 @@ If you are **not** using the [QuickBooks transformation package](https://github.
 ```yaml
 packages:
   - package: fivetran/quickbooks_source
-    version: [">=0.9.0", "<0.10.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.11.0", "<0.12.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 ### Step 3: Define database and schema variables

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 name: 'quickbooks_source'
 
-version: '0.10.1'
+version: '0.11.0'
 
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 

macros/get_account_columns.sql

@@ -19,7 +19,8 @@
     {"name": "sub_account", "datatype": "boolean"},
     {"name": "sync_token", "datatype": dbt.type_string()},
     {"name": "tax_code_id", "datatype": dbt.type_string()},
-    {"name": "updated_at", "datatype": dbt.type_timestamp()}
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"}
 ] %}
 
 {{ return(columns) }}

macros/get_bundle_columns.sql

@@ -13,7 +13,8 @@
     {"name": "sync_token", "datatype": dbt.type_string()},
     {"name": "taxable", "datatype": "boolean"},
     {"name": "unit_price", "datatype": dbt.type_float()},
-    {"name": "updated_at", "datatype": dbt.type_timestamp()}
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"}
 ] %}
 
 {{ return(columns) }}

macros/get_customer_columns.sql

@@ -40,7 +40,8 @@
     {"name": "taxable", "datatype": "boolean"},
     {"name": "title", "datatype": dbt.type_string()},
     {"name": "updated_at", "datatype": dbt.type_timestamp()},
-    {"name": "website", "datatype": dbt.type_string()}
+    {"name": "website", "datatype": dbt.type_string()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"}
 ] %}
 
 {{ return(columns) }}

macros/get_department_columns.sql

@@ -10,7 +10,8 @@
     {"name": "parent_department_id", "datatype": dbt.type_string()},
     {"name": "sub_department", "datatype": "boolean"},
     {"name": "sync_token", "datatype": dbt.type_string()},
-    {"name": "updated_at", "datatype": dbt.type_timestamp()}
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"}
 ] %}
 
 {{ return(columns) }}

macros/get_item_columns.sql

@@ -28,7 +28,8 @@
     {"name": "track_quantity_on_hand", "datatype": "boolean"},
     {"name": "type", "datatype": dbt.type_string()},
     {"name": "unit_price", "datatype": dbt.type_float()},
-    {"name": "updated_at", "datatype": dbt.type_timestamp()}
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"}
 ] %}
 
 {{ return(columns) }}

macros/get_vendor_columns.sql

@@ -28,7 +28,8 @@
     {"name": "title", "datatype": dbt.type_string()},
     {"name": "updated_at", "datatype": dbt.type_timestamp()},
     {"name": "vendor_1099", "datatype": "boolean"},
-    {"name": "web_url", "datatype": dbt.type_string()}
+    {"name": "web_url", "datatype": dbt.type_string()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"}
 ] %}
 
 {{ return(columns) }}

models/src_quickbooks.yml

@@ -47,6 +47,8 @@ sources:
             description: Specifies the balance amount for the current Account sub accounts.
           - name: updated_at
             description: Timestamp of the date which the account was last updated.
+          - name: _fivetran_deleted
+            description: "{{ doc('_fivetran_deleted') }}" 
 
       - name: address
         identifier: "{{ var('quickbooks_address_identifier', 'address') }}"
@@ -215,6 +217,8 @@ sources:
             description: Full name given to the bundle.
           - name: updated_at
             description: Timestamp of the date when the bundle was last updated.
+          - name: _fivetran_deleted
+            description: "{{ doc('_fivetran_deleted') }}"
 
       - name: credit_card_payment_txn
         identifier: "{{ var('quickbooks_credit_card_payment_txn_identifier', 'credit_card_payment_txn') }}"
@@ -319,6 +323,8 @@ sources:
             description: The website url associated with the customer.
           - name: taxable
             description: Boolean indicating whether the customer is taxable.
+          - name: _fivetran_deleted
+            description: "{{ doc('_fivetran_deleted') }}"
 
       - name: department
         identifier: "{{ var('quickbooks_department_identifier', 'department') }}"
@@ -340,6 +346,8 @@ sources:
             description: Boolean indicating whether the department is a sub department.
           - name: parent_department_id
             description: "Reference to the {{ doc('id') }} parent department if sub_department = true."
+          - name: _fivetran_deleted
+            description: "{{ doc('_fivetran_deleted') }}"
 
       - name: deposit_line
         identifier: "{{ var('quickbooks_deposit_line_identifier', 'deposit_line') }}"
@@ -593,6 +601,8 @@ sources:
             description: Classification that specifies the use of this item.
           - name: unit_price
             description: The monetary value of the service or product, as expressed in the home currency.
+          - name: _fivetran_deleted
+            description: "{{ doc('_fivetran_deleted') }}"
 
       - name: journal_entry_line
         identifier: "{{ var('quickbooks_journal_entry_line_identifier', 'journal_entry_line') }}"
@@ -1062,3 +1072,5 @@ sources:
             description: Website url associated with the vendor.
           - name: vendor_1099
             description: Boolean indicating whether this vendor is an independent contractor; someone who is given a 1099-MISC form at the end of the year.
+          - name: _fivetran_deleted
+            description: "{{ doc('_fivetran_deleted') }}"

models/stg_quickbooks.yml

@@ -48,6 +48,8 @@ models:
         description: Specifies the balance amount for the current Account sub accounts.
       - name: updated_at
         description: Timestamp of the date which the account was last updated.
+      - name: _fivetran_deleted
+        description: "{{ doc('_fivetran_deleted') }}" 
 
   - name: stg_quickbooks__address
     description: "{{ doc('address_table') }}"

models/stg_quickbooks__account.sql

@@ -49,9 +49,12 @@ final as (
         description,
         fully_qualified_name,
         updated_at,
-        source_relation
+        source_relation,
+        _fivetran_deleted
+
     from account
 )
 
 select *
-from final
+from final
+where not coalesce(_fivetran_deleted, false)

models/stg_quickbooks__bundle.sql

@@ -42,9 +42,12 @@ final as (
         active as is_active,
         fully_qualified_name,
         updated_at,
-        source_relation
+        source_relation,
+        _fivetran_deleted
 
     from fields
 )
 
-select * from final
+select * 
+from final
+where not coalesce(_fivetran_deleted, false)

models/stg_quickbooks__customer.sql

@@ -46,11 +46,12 @@ final as (
         display_name,
         website,
         taxable,
-        source_relation
-
+        source_relation,
+        _fivetran_deleted
 
     from fields
 )
 
 select * 
 from final
+where not coalesce(_fivetran_deleted, false)

models/stg_quickbooks__department.sql

@@ -46,9 +46,12 @@ final as (
         name,
         sub_department as is_sub_department,
         parent_department_id,
-        source_relation
+        source_relation,
+        _fivetran_deleted
+
     from fields
 )
 
 select *
 from final
+where not coalesce(_fivetran_deleted, false)

models/stg_quickbooks__item.sql

@@ -47,10 +47,12 @@ final as (
         unit_price,
         inventory_start_date,
         cast(parent_item_id as {{ dbt.type_string() }}) as parent_item_id,
-        source_relation
+        source_relation,
+        _fivetran_deleted
 
     from fields
 )
 
 select * 
 from final
+where not coalesce(_fivetran_deleted, false)

models/stg_quickbooks__vendor.sql

@@ -46,9 +46,12 @@ final as (
         display_name,
         web_url,
         vendor_1099 as is_vendor_1099,
-        source_relation
+        source_relation,
+        _fivetran_deleted
+
     from fields
 )
 
 select * 
 from final
+where not coalesce(_fivetran_deleted, false)