models updated

gnosischain · Nov 2, 2024 · 02a41ce · 02a41ce
1 parent 395b3ed
commit 02a41ce
Show file tree

Hide file tree

Showing 18 changed files with 300 additions and 343 deletions.
diff --git a/cron.sh b/cron.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-dbt run -s execution_power
+dbt run 
 
 
 
diff --git a/cron_preview.sh b/cron_preview.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-dbt run -s execution_power
+dbt run 
 
 
 
diff --git a/macros/db/get_incremental_filter.sql b/macros/db/get_incremental_filter.sql
@@ -0,0 +1,14 @@
+{% macro get_incremental_filter() %}
+    {% if is_incremental() %}
+    last_partition AS (
+        SELECT max(partition_month) as partition_month
+        FROM {{ this }}
+    ),
+    {% endif %}
+{% endmacro %}
+
+{% macro apply_incremental_filter(timestamp_field, add_and=false) %}
+    {% if is_incremental() %}
+    {{ "AND " if add_and else "WHERE "}}toStartOfMonth({{ timestamp_field }}) >= (SELECT partition_month FROM last_partition)
+    {% endif %}
+{% endmacro %}
diff --git a/models/consensus/attestations/metrics/consensus_atts_inclusion_distance.sql b/models/consensus/attestations/metrics/consensus_atts_inclusion_distance.sql
@@ -0,0 +1,35 @@
+{{ 
+    config(
+        materialized='incremental',
+        incremental_strategy='delete+insert',
+        engine='ReplacingMergeTree()',
+        order_by='(day, inc_dist_cohort)',
+        unique_key='(day, inc_dist_cohort)',
+        partition_by='partition_month'
+    ) 
+}}
+
+WITH 
+
+{{ get_incremental_filter() }}
+
+
+inclusion_distance AS (
+    SELECT 
+        toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) AS day
+        ,f_inclusion_slot - f_slot AS inc_dist_cohort
+        ,COUNT(*) AS cnt
+    FROM  {{ get_postgres('chaind', 't_attestations') }}
+    {{ apply_incremental_filter(compute_timestamp_at_slot('f_inclusion_slot')) }}
+    GROUP BY 1,2
+)
+
+SELECT 
+    toStartOfMonth(day) AS partition_month
+    ,day
+    ,inc_dist_cohort
+    ,cnt
+FROM 
+    inclusion_distance
+WHERE
+    day < (SELECT MAX(day) FROM inclusion_distance)
diff --git a/models/consensus/attestations/metrics/consensus_atts_inclusion_distance_d.sql b/models/consensus/attestations/metrics/consensus_atts_inclusion_distance_d.sql
diff --git a/models/consensus/attestations/metrics/schema.yml b/models/consensus/attestations/metrics/schema.yml
@@ -1,6 +1,6 @@
 version: 2
 models:
-- name: consensus_atts_inclusion_distance_d
+- name: consensus_atts_inclusion_distance
   meta:
     blockchain: consensus
     sector: attestations
@@ -9,7 +9,7 @@ models:
     tags:
     - consensus
     - attestations
-  description: 'TODO: Add description for consensus_atts_inclusion_distance_d'
+  description: 'TODO: Add description for consensus_atts_inclusion_distance'
   columns:
   - name: cnt
     description: 'TODO: Add description for cnt'
@@ -26,11 +26,6 @@ models:
     data_tests:
     - not_null
     - unique
-  - name: partition_month
-    description: 'TODO: Add description for partition_month'
-    data_tests:
-    - not_null
-    - unique
   data_tests:
   - dbt_utils.unique_combination_of_columns:
       combination_of_columns:

diff --git a/...ics/consensus_blocks_graffiti_top10_d.sql → ...trics/consensus_blocks_graffiti_top10.sql b/...ics/consensus_blocks_graffiti_top10_d.sql → ...trics/consensus_blocks_graffiti_top10.sql
@@ -4,7 +4,7 @@
        incremental_strategy='delete+insert',
        engine='ReplacingMergeTree()',
        order_by='(day, graffiti_top10)',
-       primary_key='(day, graffiti_top10)',
+       unique_key='(day, graffiti_top10)',
        partition_by='partition_month',
        settings={
            "allow_nullable_key": 1

diff --git a/models/consensus/blocks/metrics/consensus_blocks_production.sql b/models/consensus/blocks/metrics/consensus_blocks_production.sql
@@ -1,16 +1,25 @@
 {{ 
     config(
         materialized='incremental',
-        incremental_strategy='insert_overwrite',
-        partition_by='toStartOfWeek(day)'
+        incremental_strategy='delete+insert',
+        engine='ReplacingMergeTree()',
+        order_by='(day, label)',
+        unique_key='(day, label)',
+        partition_by='partition_month'
     ) 
 }}
 
-WITH genesis AS (
+WITH 
+
+{{ get_incremental_filter() }}
+
+
+genesis AS (
     SELECT f_time AS genesis_time
     FROM {{ get_postgres('chaind', 't_genesis') }}
     LIMIT 1
 ),
+
 blocks AS (
     SELECT
         toDate({{ compute_timestamp_at_slot('f_slot') }}) AS day,
@@ -19,40 +28,51 @@ blocks AS (
         CAST(SUM(IF(f_canonical, 1, 0)) AS Int64) AS proposed
     FROM
         {{ get_postgres('chaind', 't_blocks') }}
-    {% if is_incremental() %}
-    WHERE toStartOfWeek(toDate({{ compute_timestamp_at_slot('f_slot') }})) >= (SELECT max(toStartOfWeek(day)) FROM {{ this }})
-    {% endif %}
+    {{ apply_incremental_filter(compute_timestamp_at_slot('f_slot')) }}
     GROUP BY 1
 ),
+
 chain_specs AS (
     SELECT
         toInt64OrZero({{ get_chain_spec('SECONDS_PER_SLOT') }}) AS seconds_per_slot,
         {{ seconds_until_end_of_day('genesis.genesis_time') }} AS seconds_until_end_of_day
     FROM genesis
-)
+),
 
-SELECT day, forked AS cnt, 'forked' AS label 
-FROM blocks
+final AS (
+    SELECT day, forked AS cnt, 'forked' AS label 
+    FROM blocks
 
-UNION ALL
+    UNION ALL
 
-SELECT day, proposed AS cnt, 'proposed' AS label 
-FROM blocks
+    SELECT day, proposed AS cnt, 'proposed' AS label 
+    FROM blocks
 
-UNION ALL
+    UNION ALL
+
+    SELECT 
+        blocks.day, 
+        CASE 
+            WHEN blocks.f_slot_start = 0 
+                THEN CAST(
+                    (chain_specs.seconds_until_end_of_day) / chain_specs.seconds_per_slot - (blocks.proposed + blocks.forked) AS Int64
+                )
+            ELSE
+                CAST(
+                    (24 * 3600) / chain_specs.seconds_per_slot - (blocks.proposed + blocks.forked) AS Int64
+                ) 
+        END AS cnt,
+        'missed' AS label 
+    FROM blocks
+    CROSS JOIN chain_specs
+)
 
 SELECT 
-    blocks.day, 
-    CASE 
-        WHEN blocks.f_slot_start = 0 
-            THEN CAST(
-                (chain_specs.seconds_until_end_of_day) / chain_specs.seconds_per_slot - (blocks.proposed + blocks.forked) AS Int64
-            )
-        ELSE
-            CAST(
-                (24 * 3600) / chain_specs.seconds_per_slot - (blocks.proposed + blocks.forked) AS Int64
-            ) 
-    END AS cnt,
-    'missed' AS label 
-FROM blocks
-CROSS JOIN chain_specs
+    toStartOfMonth(day) AS partition_month
+    ,day
+    ,cnt
+    ,label 
+FROM
+    final
+WHERE 
+    day < (SELECT MAX(day) FROM final)
diff --git a/models/consensus/blocks/metrics/schema.yml b/models/consensus/blocks/metrics/schema.yml
@@ -26,7 +26,7 @@ models:
       combination_of_columns:
       - proposer_index
       - withdrawable_time
-- name: consensus_blocks_graffiti_top10_d
+- name: consensus_blocks_graffiti_top10
   meta:
     blockchain: consensus
     sector: blocks
@@ -35,7 +35,7 @@ models:
     tags:
     - consensus
     - blocks
-  description: 'TODO: Add description for consensus_blocks_graffiti_top10_d'
+  description: 'TODO: Add description for consensus_blocks_graffiti_top10'
   columns:
   - name: cnt
     description: 'TODO: Add description for cnt'

diff --git a/models/consensus/blocks/transformations/consensus_blocks_graffiti.sql b/models/consensus/blocks/transformations/consensus_blocks_graffiti.sql
@@ -1,28 +1,37 @@
 {{ 
-   config(
-       materialized='incremental',
-       incremental_strategy='delete+insert',
+    config(
+        materialized='incremental',
+        incremental_strategy='delete+insert',
         engine='ReplacingMergeTree()',
         order_by='(day, graffiti, f_proposer_index)',
-        primary_key='(day, graffiti, f_proposer_index)',
-       partition_by='partition_month'
-   ) 
+        unique_key='(day, graffiti, f_proposer_index)',
+        partition_by='partition_month'
+    ) 
 }}
 
+WITH 
 
-WITH blocks_graffiti AS (
+{{ get_incremental_filter() }}
+
+blocks_graffiti AS (
     SELECT
-        toStartOfMonth({{ compute_timestamp_at_slot('f_slot') }}) AS partition_month
-        ,toDate({{ compute_timestamp_at_slot('f_slot') }}) AS day
+        toDate({{ compute_timestamp_at_slot('f_slot') }}) AS day
         ,f_proposer_index
         ,{{ decode_graffiti('f_graffiti') }} AS graffiti
         ,COUNT(*) AS cnt
     FROM
        {{ get_postgres('chaind', 't_blocks') }}
-       {% if is_incremental() %}
-        WHERE toStartOfMonth({{ compute_timestamp_at_slot('f_slot') }}) >= (SELECT max(partition_month) FROM {{ this }})
-       {% endif %}
-    GROUP BY 1, 2, 3, 4
+    {{ apply_incremental_filter(compute_timestamp_at_slot('f_slot')) }}
+    GROUP BY 1, 2, 3
 )
 
-SELECT * FROM blocks_graffiti
+SELECT 
+    toStartOfMonth(day) AS partition_month
+    ,day
+    ,f_proposer_index
+    ,graffiti
+    ,cnt
+FROM 
+    blocks_graffiti
+WHERE
+    day < (SELECT MAX(day) FROM blocks_graffiti)
diff --git a/models/consensus/blocks/transformations/schema.yml b/models/consensus/blocks/transformations/schema.yml
@@ -31,11 +31,6 @@ models:
     data_tests:
     - not_null
     - unique
-  - name: partition_month
-    description: 'TODO: Add description for partition_month'
-    data_tests:
-    - not_null
-    - unique
   data_tests:
   - dbt_utils.unique_combination_of_columns:
       combination_of_columns: