From a30b443c3dfd7a8759e2fafc478a056b5820d172 Mon Sep 17 00:00:00 2001 From: hdser Date: Mon, 21 Oct 2024 15:21:35 +0200 Subject: [PATCH 1/4] first models --- macros/db/compute_timestamp_at_slot.sql | 19 ++++++ macros/db/flexible_source.sql | 23 +++++++ macros/db/get_chain_spec.sql | 3 + macros/db/seconds_until_end_of_day.sql | 7 ++ .../gnosis_carbon_emissions.sql | 0 .../gnosis_power_consumption.sql | 0 .../consensus_atts_inclusion_distance_d.sql | 24 +++++++ ...forks.sql => gnosis_p2p_nodes_forks.sqlxx} | 0 ...des_geo.sql => gnosis_p2p_nodes_geo.sqlxx} | 0 ...s_host.sql => gnosis_p2p_nodes_host.sqlxx} | 0 .../metrics/p2p/p2p_valtrack_nodes_cnt_1d.sql | 1 + .../p2p/p2p_valtrack_nodes_cnt_1h.sql | 0 .../p2p/p2p_valtrack_nodes_cnt_country_1d.sql | 0 ...ql => p2p_valtrack_nodes_geo_last_day.sql} | 4 +- .../consensus_validators_activations.sql | 53 +++++++++++++++ .../consensus_validators_participation.sql | 34 ++++++++++ ...onsensus_validators_participation_rate.sql | 40 ++++++++++++ .../consensus_atts_inclusion_distance.sql | 65 +++++++++++++++++++ ...consensus_atts_inclusion_distance_old.sql} | 21 ++++-- .../consensus_blocks_production.sql | 58 +++++++++++++++++ models/transformations/p2p/test2.sql | 7 ++ .../consensus_validators_status.sql | 25 +++++++ 22 files changed, 375 insertions(+), 9 deletions(-) create mode 100644 macros/db/compute_timestamp_at_slot.sql create mode 100644 macros/db/flexible_source.sql create mode 100644 macros/db/get_chain_spec.sql create mode 100644 macros/db/seconds_until_end_of_day.sql rename models/metrics/{carbon_ratings => _carbon_ratings}/gnosis_carbon_emissions.sql (100%) rename models/metrics/{carbon_ratings => _carbon_ratings}/gnosis_power_consumption.sql (100%) create mode 100644 models/metrics/attestations/consensus_atts_inclusion_distance_d.sql rename models/metrics/p2p/{gnosis_p2p_nodes_forks.sql => gnosis_p2p_nodes_forks.sqlxx} (100%) rename models/metrics/p2p/{gnosis_p2p_nodes_geo.sql => gnosis_p2p_nodes_geo.sqlxx} (100%) rename models/metrics/p2p/{gnosis_p2p_nodes_host.sql => gnosis_p2p_nodes_host.sqlxx} (100%) create mode 100644 models/metrics/p2p/p2p_valtrack_nodes_cnt_1d.sql rename models/{transformations => metrics}/p2p/p2p_valtrack_nodes_cnt_1h.sql (100%) rename models/{transformations => metrics}/p2p/p2p_valtrack_nodes_cnt_country_1d.sql (100%) rename models/metrics/p2p/{gnosis_p2p_nodes_geo_last_day.sql => p2p_valtrack_nodes_geo_last_day.sql} (78%) create mode 100644 models/metrics/validators/consensus_validators_activations.sql create mode 100644 models/metrics/validators/consensus_validators_participation.sql create mode 100644 models/metrics/validators/consensus_validators_participation_rate.sql create mode 100644 models/transformations/attestations/consensus_atts_inclusion_distance.sql rename models/transformations/attestations/{gnosis_consensus_attestations.sql => consensus_atts_inclusion_distance_old.sql} (65%) create mode 100644 models/transformations/attestations/consensus_blocks_production.sql create mode 100644 models/transformations/p2p/test2.sql create mode 100644 models/transformations/validatros/consensus_validators_status.sql diff --git a/macros/db/compute_timestamp_at_slot.sql b/macros/db/compute_timestamp_at_slot.sql new file mode 100644 index 0000000..62663fe --- /dev/null +++ b/macros/db/compute_timestamp_at_slot.sql @@ -0,0 +1,19 @@ +{% macro compute_timestamp_at_slot(slot) %} + +addSeconds( + (SELECT f_time FROM {{ get_postgres('gnosis_chaind', 't_genesis') }} LIMIT 1), + ({{ slot }} - 0) * (SELECT toInt32(f_value) FROM {{ get_postgres('gnosis_chaind', 't_chain_spec') }} WHERE f_key = 'SECONDS_PER_SLOT' LIMIT 1) +) + +{% endmacro %} + + +{% macro compute_timestamp_at_epoch(epoch) %} + +addSeconds( + (SELECT f_time FROM {{ get_postgres('gnosis_chaind', 't_genesis') }} LIMIT 1), + ({{ epoch }} - 0) * (SELECT toInt32(f_value) FROM {{ get_postgres('gnosis_chaind', 't_chain_spec') }} WHERE f_key = 'SECONDS_PER_SLOT' LIMIT 1) + *(SELECT toInt32(f_value) FROM {{ get_postgres('gnosis_chaind', 't_chain_spec') }} WHERE f_key = 'SLOTS_PER_EPOCH' LIMIT 1) +) + +{% endmacro %} \ No newline at end of file diff --git a/macros/db/flexible_source.sql b/macros/db/flexible_source.sql new file mode 100644 index 0000000..1ff7456 --- /dev/null +++ b/macros/db/flexible_source.sql @@ -0,0 +1,23 @@ +{% macro flexible_source(schema, table, env='dev') %} + {% if env not in ['dev', 'prod'] %} + {{ exceptions.raise_compiler_error("Invalid environment. Choose either 'dev' or 'prod'.") }} + {% endif %} + + {% set env_prefix = 'CLICKHOUSE_' ~ env | upper ~ '_' %} + + {% set env_url = env_var(env_prefix ~ 'URL', '') %} + {% set env_port = '9440' %} + {% set env_user = env_var(env_prefix ~ 'USER', '') %} + {% set env_password = env_var(env_prefix ~ 'PASSWORD', '') %} + + {% if env_url and env_url != env_var('CLICKHOUSE_URL') %} + {% set remote_source = 'remoteSecure(\'' ~ env_url ~ ':' ~ env_port ~ '\', \'' ~ schema ~ '\', \'' ~ table ~ '\', \'' ~ env_user ~ '\', \'' ~ env_password ~ '\') + SETTINGS + connect_timeout_with_failover_secure_ms = 60000, + receive_timeout = 120000, + send_timeout = 120000 ' %} + {{ remote_source }} + {% else %} + {{ source(schema, table) }} + {% endif %} +{% endmacro %} diff --git a/macros/db/get_chain_spec.sql b/macros/db/get_chain_spec.sql new file mode 100644 index 0000000..611de63 --- /dev/null +++ b/macros/db/get_chain_spec.sql @@ -0,0 +1,3 @@ +{% macro get_chain_spec(spec_key) %} +(SELECT f_value FROM {{ get_postgres('gnosis_chaind', 't_chain_spec') }} WHERE f_key = '{{ spec_key }}' LIMIT 1) +{% endmacro %} \ No newline at end of file diff --git a/macros/db/seconds_until_end_of_day.sql b/macros/db/seconds_until_end_of_day.sql new file mode 100644 index 0000000..2b1900b --- /dev/null +++ b/macros/db/seconds_until_end_of_day.sql @@ -0,0 +1,7 @@ +{% macro seconds_until_end_of_day(timestamp_column) %} +( + 86400 - ( + toUInt32(modulo(toUnixTimestamp({{ timestamp_column }}), 86400)) + ) +) +{% endmacro %} \ No newline at end of file diff --git a/models/metrics/carbon_ratings/gnosis_carbon_emissions.sql b/models/metrics/_carbon_ratings/gnosis_carbon_emissions.sql similarity index 100% rename from models/metrics/carbon_ratings/gnosis_carbon_emissions.sql rename to models/metrics/_carbon_ratings/gnosis_carbon_emissions.sql diff --git a/models/metrics/carbon_ratings/gnosis_power_consumption.sql b/models/metrics/_carbon_ratings/gnosis_power_consumption.sql similarity index 100% rename from models/metrics/carbon_ratings/gnosis_power_consumption.sql rename to models/metrics/_carbon_ratings/gnosis_power_consumption.sql diff --git a/models/metrics/attestations/consensus_atts_inclusion_distance_d.sql b/models/metrics/attestations/consensus_atts_inclusion_distance_d.sql new file mode 100644 index 0000000..edd874c --- /dev/null +++ b/models/metrics/attestations/consensus_atts_inclusion_distance_d.sql @@ -0,0 +1,24 @@ +{{ + config( + materialized='incremental', + incremental_strategy='insert_overwrite', + partition_by='day' + ) +}} + +WITH + + +inclusion_distance AS ( + SELECT + toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) AS day + ,inc_dist_cohort + ,SUM(cnt) AS cnt + FROM {{ ref('consensus_atts_inclusion_distance') }} + {% if is_incremental() %} + WHERE toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) >= (SELECT max(day) FROM {{ this }}) + {% endif %} + GROUP BY 1,2 +) + +SELECT * FROM inclusion_distance diff --git a/models/metrics/p2p/gnosis_p2p_nodes_forks.sql b/models/metrics/p2p/gnosis_p2p_nodes_forks.sqlxx similarity index 100% rename from models/metrics/p2p/gnosis_p2p_nodes_forks.sql rename to models/metrics/p2p/gnosis_p2p_nodes_forks.sqlxx diff --git a/models/metrics/p2p/gnosis_p2p_nodes_geo.sql b/models/metrics/p2p/gnosis_p2p_nodes_geo.sqlxx similarity index 100% rename from models/metrics/p2p/gnosis_p2p_nodes_geo.sql rename to models/metrics/p2p/gnosis_p2p_nodes_geo.sqlxx diff --git a/models/metrics/p2p/gnosis_p2p_nodes_host.sql b/models/metrics/p2p/gnosis_p2p_nodes_host.sqlxx similarity index 100% rename from models/metrics/p2p/gnosis_p2p_nodes_host.sql rename to models/metrics/p2p/gnosis_p2p_nodes_host.sqlxx diff --git a/models/metrics/p2p/p2p_valtrack_nodes_cnt_1d.sql b/models/metrics/p2p/p2p_valtrack_nodes_cnt_1d.sql new file mode 100644 index 0000000..d03dcb2 --- /dev/null +++ b/models/metrics/p2p/p2p_valtrack_nodes_cnt_1d.sql @@ -0,0 +1 @@ +{{ valtack_nodes_activity(resolution='day') }} \ No newline at end of file diff --git a/models/transformations/p2p/p2p_valtrack_nodes_cnt_1h.sql b/models/metrics/p2p/p2p_valtrack_nodes_cnt_1h.sql similarity index 100% rename from models/transformations/p2p/p2p_valtrack_nodes_cnt_1h.sql rename to models/metrics/p2p/p2p_valtrack_nodes_cnt_1h.sql diff --git a/models/transformations/p2p/p2p_valtrack_nodes_cnt_country_1d.sql b/models/metrics/p2p/p2p_valtrack_nodes_cnt_country_1d.sql similarity index 100% rename from models/transformations/p2p/p2p_valtrack_nodes_cnt_country_1d.sql rename to models/metrics/p2p/p2p_valtrack_nodes_cnt_country_1d.sql diff --git a/models/metrics/p2p/gnosis_p2p_nodes_geo_last_day.sql b/models/metrics/p2p/p2p_valtrack_nodes_geo_last_day.sql similarity index 78% rename from models/metrics/p2p/gnosis_p2p_nodes_geo_last_day.sql rename to models/metrics/p2p/p2p_valtrack_nodes_geo_last_day.sql index 3b1a414..05f01f3 100644 --- a/models/metrics/p2p/gnosis_p2p_nodes_geo_last_day.sql +++ b/models/metrics/p2p/p2p_valtrack_nodes_geo_last_day.sql @@ -21,6 +21,6 @@ INNER JOIN {{ source('valtrack','ip_metadata') }} t2 ON t2.ip = t3.ip WHERE - t1.timestamp >= date_trunc('day', now() - INTERVAL 5 DAY) + t1.timestamp >= date_trunc('day', now() - INTERVAL 14 DAY) AND - t1.timestamp < date_trunc('day', now()- INTERVAL 4 DAY) \ No newline at end of file + t1.timestamp < date_trunc('day', now()- INTERVAL 13 DAY) \ No newline at end of file diff --git a/models/metrics/validators/consensus_validators_activations.sql b/models/metrics/validators/consensus_validators_activations.sql new file mode 100644 index 0000000..5662e91 --- /dev/null +++ b/models/metrics/validators/consensus_validators_activations.sql @@ -0,0 +1,53 @@ +{{ + config( + materialized='incremental', + incremental_strategy='insert_overwrite', + partition_by='toYYYYMM(coalesce(month, toDate(\'1970-01-01\')))' + ) +}} + +WITH validators_activations AS ( + SELECT + toDate(activation_time) AS day, + toStartOfMonth(toDate(activation_time)) AS month, + CAST(COUNT(*) AS Int64) AS cnt, + 'activations' AS label + FROM {{ ref('consensus_validators_status') }} + {% if is_incremental() %} + WHERE toDate(activation_time) >= (SELECT max(day) FROM {{ this }}) + {% endif %} + GROUP BY 1, 2 +), + +validators_exits AS ( + SELECT + toDate(withdrawable_time) AS day, + toStartOfMonth(toDate(withdrawable_time)) AS month, + -CAST(COUNT(*) AS Int64) AS cnt, + 'exits' AS label + FROM {{ ref('consensus_validators_status') }} + WHERE + exit_time IS NOT NULL + {% if is_incremental() %} + AND toDate(exit_time) >= (SELECT max(day) FROM {{ this }}) + {% endif %} + GROUP BY 1, 2 +) + +SELECT + day, + coalesce(month, toDate('1970-01-01')) AS month, + cnt, + label +FROM validators_activations + +UNION ALL + +SELECT + day, + coalesce(month, toDate('1970-01-01')) AS month, + cnt, + label +FROM validators_exits + +ORDER BY day, label \ No newline at end of file diff --git a/models/metrics/validators/consensus_validators_participation.sql b/models/metrics/validators/consensus_validators_participation.sql new file mode 100644 index 0000000..406fc52 --- /dev/null +++ b/models/metrics/validators/consensus_validators_participation.sql @@ -0,0 +1,34 @@ +{{ + config( + materialized='incremental', + incremental_strategy='insert_overwrite', + partition_by=['day'], + engine='MergeTree()', + order_by='day' + ) +}} + +WITH chunked_data AS ( + SELECT + toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) AS day, + f_aggregation_indices, + intDiv(rowNumberInAllBlocks(), 10000) AS chunk + FROM {{ get_postgres('gnosis_chaind', 't_attestations') }} + {% if is_incremental() %} + WHERE toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) >= (SELECT max(day) FROM {{ this }}) + {% endif %} +), +chunked_aggregation AS ( + SELECT + day, + chunk, + arrayDistinct(arrayFlatten(groupArrayArray(f_aggregation_indices))) AS chunk_distinct_array + FROM chunked_data + GROUP BY day, chunk +) + +SELECT + day, + length(arrayDistinct(arrayFlatten(groupArrayArray(chunk_distinct_array)))) AS distinct_count +FROM chunked_aggregation +GROUP BY day \ No newline at end of file diff --git a/models/metrics/validators/consensus_validators_participation_rate.sql b/models/metrics/validators/consensus_validators_participation_rate.sql new file mode 100644 index 0000000..26a2b2a --- /dev/null +++ b/models/metrics/validators/consensus_validators_participation_rate.sql @@ -0,0 +1,40 @@ +{{ + config( + materialized='incremental', + incremental_strategy='insert_overwrite', + partition_by=['day'], + engine='MergeTree()', + order_by='day' + ) +}} + +WITH + +participation AS ( + SELECT + day, + distinct_count AS active + FROM {{ ref('consensus_validators_participation') }} +), + +validators AS ( + SELECT + day, + SUM(cnt) OVER (ORDER BY day) AS total_validators + FROM {{ ref('consensus_validators_activations') }} +) + +SELECT + p.day AS day, + p.active, + v.total_validators, + CASE + WHEN v.total_validators > 0 THEN p.active / v.total_validators + ELSE NULL + END AS participation_rate +FROM + participation p +INNER JOIN + validators v + ON p.day = v.day + diff --git a/models/transformations/attestations/consensus_atts_inclusion_distance.sql b/models/transformations/attestations/consensus_atts_inclusion_distance.sql new file mode 100644 index 0000000..b678b3b --- /dev/null +++ b/models/transformations/attestations/consensus_atts_inclusion_distance.sql @@ -0,0 +1,65 @@ +{{ + config( + materialized='incremental', + incremental_strategy='delete+insert', + engine='ReplacingMergeTree()', + order_by='(f_inclusion_slot, inc_dist_cohort)', + primary_key='(f_inclusion_slot, inc_dist_cohort)' + ) +}} + +WITH + +attestations AS ( + SELECT + f_inclusion_slot, + f_slot, + f_inclusion_index + FROM + {{ get_postgres('gnosis_chaind', 't_attestations') }} + {% if is_incremental() %} + WHERE f_inclusion_slot > (SELECT max(f_inclusion_slot) FROM {{ this }}) + {% endif %} +), + +proposed_slots AS ( + SELECT + f_slot + FROM + {{ get_postgres('gnosis_chaind', 't_blocks') }} + {% if is_incremental() %} + WHERE f_slot > (SELECT min(f_slot) FROM attestations) + AND f_slot <= (SELECT max(f_inclusion_slot) FROM attestations) + {% endif %} +), + +slot_ranges AS ( + SELECT + a.f_inclusion_slot, + a.f_slot AS attestation_slot, + a.f_inclusion_index, + arrayJoin(range(a.f_slot + 1, a.f_inclusion_slot + 1)) AS slot + FROM attestations a +), + +inclusion_distance AS ( + SELECT + sr.f_inclusion_slot, + sr.attestation_slot, + sr.f_inclusion_index, + countDistinct(ps.f_slot) AS inc_dist_cohort + FROM slot_ranges sr + LEFT JOIN proposed_slots ps ON sr.slot = ps.f_slot + GROUP BY + sr.f_inclusion_slot, + sr.attestation_slot, + sr.f_inclusion_index +) + +SELECT + f_inclusion_slot, + inc_dist_cohort, + COUNT(*) AS cnt +FROM + inclusion_distance +GROUP BY 1, 2 \ No newline at end of file diff --git a/models/transformations/attestations/gnosis_consensus_attestations.sql b/models/transformations/attestations/consensus_atts_inclusion_distance_old.sql similarity index 65% rename from models/transformations/attestations/gnosis_consensus_attestations.sql rename to models/transformations/attestations/consensus_atts_inclusion_distance_old.sql index 887d3e9..b38eede 100644 --- a/models/transformations/attestations/gnosis_consensus_attestations.sql +++ b/models/transformations/attestations/consensus_atts_inclusion_distance_old.sql @@ -1,7 +1,8 @@ {{ config( - materialized='table', - engine='MergeTree()', + materialized='incremental', + incremental_strategy='delete+insert', + engine='ReplacingMergeTree()', order_by='(f_inclusion_slot, inc_dist_cohort)', primary_key='(f_inclusion_slot, inc_dist_cohort)' ) @@ -14,7 +15,9 @@ total_slots AS ( f_slot FROM {{ get_postgres('gnosis_chaind', 't_proposer_duties') }} - LIMIT 100 + {% if is_incremental() %} + WHERE f_slot > (SELECT max(f_inclusion_slot) FROM {{ this }}) + {% endif %} ), proposed_slots AS ( @@ -22,7 +25,9 @@ proposed_slots AS ( f_slot FROM {{ get_postgres('gnosis_chaind', 't_blocks') }} - LIMIT 100 + {% if is_incremental() %} + WHERE f_slot > (SELECT max(f_inclusion_slot) FROM {{ this }}) + {% endif %} ), attestations AS ( @@ -32,7 +37,9 @@ attestations AS ( f_inclusion_index FROM {{ get_postgres('gnosis_chaind', 't_attestations') }} - LIMIT 100 + {% if is_incremental() %} + WHERE f_inclusion_slot > (SELECT max(f_inclusion_slot) FROM {{ this }}) + {% endif %} ), inclusion_distance AS ( @@ -55,5 +62,5 @@ SELECT inc_dist_cohort, COUNT(*) AS cnt FROM - inclusion_distance -GROUP BY 1, 2 \ No newline at end of file + inclusion_distance +GROUP BY 1, 2 diff --git a/models/transformations/attestations/consensus_blocks_production.sql b/models/transformations/attestations/consensus_blocks_production.sql new file mode 100644 index 0000000..7665c71 --- /dev/null +++ b/models/transformations/attestations/consensus_blocks_production.sql @@ -0,0 +1,58 @@ +{{ + config( + materialized='incremental', + incremental_strategy='insert_overwrite', + partition_by='day' + ) +}} + +WITH genesis AS ( + SELECT f_time AS genesis_time + FROM {{ get_postgres('gnosis_chaind', 't_genesis') }} + LIMIT 1 +), +blocks AS ( + SELECT + toDate({{ compute_timestamp_at_slot('f_slot') }}) AS day, + MIN(f_slot) AS f_slot_start, + CAST(SUM(IF(NOT f_canonical, 1, 0)) AS Int64) AS forked, + CAST(SUM(IF(f_canonical IS NULL, 1, 0)) AS Int64) AS proposed + FROM + {{ get_postgres('gnosis_chaind', 't_blocks') }} + {% if is_incremental() %} + WHERE toDate({{ compute_timestamp_at_slot('f_slot') }}) >= (SELECT max(day) FROM {{ this }}) + {% endif %} + GROUP BY 1 +), +chain_specs AS ( + SELECT + toInt64OrZero({{ get_chain_spec('SECONDS_PER_SLOT') }}) AS seconds_per_slot, + {{ seconds_until_end_of_day('genesis.genesis_time') }} AS seconds_until_end_of_day + FROM genesis +) + +SELECT day, forked AS cnt, 'forked' AS label +FROM blocks + +UNION ALL + +SELECT day, proposed AS cnt, 'proposed' AS label +FROM blocks + +UNION ALL + +SELECT + blocks.day, + CASE + WHEN blocks.f_slot_start = 0 + THEN CAST( + (chain_specs.seconds_until_end_of_day) / chain_specs.seconds_per_slot - (blocks.proposed + blocks.forked) AS Int64 + ) + ELSE + CAST( + (24 * 3600) / chain_specs.seconds_per_slot - (blocks.proposed + blocks.forked) AS Int64 + ) + END AS cnt, + 'missed' AS label +FROM blocks +CROSS JOIN chain_specs \ No newline at end of file diff --git a/models/transformations/p2p/test2.sql b/models/transformations/p2p/test2.sql new file mode 100644 index 0000000..37dcac8 --- /dev/null +++ b/models/transformations/p2p/test2.sql @@ -0,0 +1,7 @@ + +{{ config( + materialized='table' +) }} + +SELECT * +FROM {{ flexible_source('valtrack', 'ip_metadata', 'dev') }} \ No newline at end of file diff --git a/models/transformations/validatros/consensus_validators_status.sql b/models/transformations/validatros/consensus_validators_status.sql new file mode 100644 index 0000000..57a1851 --- /dev/null +++ b/models/transformations/validatros/consensus_validators_status.sql @@ -0,0 +1,25 @@ +{{ + config( + materialized='incremental', + incremental_strategy='delete+insert' + ) +}} + +WITH + +validators AS ( + SELECT + f_index + ,f_withdrawal_credentials + ,{{ compute_timestamp_at_epoch('f_activation_eligibility_epoch') }} AS activation_eligibility_time + ,{{ compute_timestamp_at_epoch('f_activation_epoch') }} AS activation_time + ,{{ compute_timestamp_at_epoch('f_exit_epoch') }} AS exit_time + ,{{ compute_timestamp_at_epoch('f_withdrawable_epoch') }} AS withdrawable_time + FROM + {{ get_postgres('gnosis_chaind', 't_validators') }} + {% if is_incremental() %} + WHERE f_inclusion_slot > (SELECT max(f_inclusion_slot) FROM {{ this }}) + {% endif %} +) + +SELECT * FROM validators \ No newline at end of file From cc504db50a843b9918f44c8e0943247a4edfb95f Mon Sep 17 00:00:00 2001 From: hdser Date: Wed, 23 Oct 2024 11:18:02 +0200 Subject: [PATCH 2/4] add grafana dashboards --- .gitignore | 1 + grafana/grafana_dashboards.sh | 16 + .../a0defb5e-33db-41da-ba04-17d698577b72.json | 632 +++++++++ .../b0e1d65b-c5d9-4f58-9e2b-9c29779c2c53.json | 1168 +++++++++++++++++ .../c049b211-0058-4301-b9e2-aa54973022e1.json | 48 + .../h73c933j-09fc-095c-9185-26e92003e016.json | 412 ++++++ .../consensus_validators_waiting_times.sql | 30 + models/sources/p2p/p2p_sources.yml | 9 + .../consensus_validators_status.sql | 0 .../validators/test_chaind.sql | 16 + profiles.yml | 10 + 11 files changed, 2342 insertions(+) create mode 100755 grafana/grafana_dashboards.sh create mode 100644 grafana/grafana_dashboards_backup/a0defb5e-33db-41da-ba04-17d698577b72.json create mode 100644 grafana/grafana_dashboards_backup/b0e1d65b-c5d9-4f58-9e2b-9c29779c2c53.json create mode 100644 grafana/grafana_dashboards_backup/c049b211-0058-4301-b9e2-aa54973022e1.json create mode 100644 grafana/grafana_dashboards_backup/h73c933j-09fc-095c-9185-26e92003e016.json create mode 100644 models/metrics/validators/consensus_validators_waiting_times.sql rename models/transformations/{validatros => validators}/consensus_validators_status.sql (100%) create mode 100644 models/transformations/validators/test_chaind.sql diff --git a/.gitignore b/.gitignore index 654500b..5d36ead 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ # environments .env +**/.env .user.yml # vscode config diff --git a/grafana/grafana_dashboards.sh b/grafana/grafana_dashboards.sh new file mode 100755 index 0000000..64ae39e --- /dev/null +++ b/grafana/grafana_dashboards.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Load environment variables from .env file +source .env + +# Create a folder to store your backups +BACKUP_DIR="./grafana_dashboards_backup" +mkdir -p $BACKUP_DIR + +# Get the list of all dashboards +curl -H "Authorization: Bearer $API_TOKEN" "$GRAFANA_URL/api/search?query=&" | jq '.[] | .uid' | sed 's/"//g' | while read dashboard_uid +do + # Download the dashboard by UID + echo "Downloading dashboard UID: $dashboard_uid" + curl -H "Authorization: Bearer $API_TOKEN" "$GRAFANA_URL/api/dashboards/uid/$dashboard_uid" | jq '.' > "$BACKUP_DIR/$dashboard_uid.json" +done diff --git a/grafana/grafana_dashboards_backup/a0defb5e-33db-41da-ba04-17d698577b72.json b/grafana/grafana_dashboards_backup/a0defb5e-33db-41da-ba04-17d698577b72.json new file mode 100644 index 0000000..ff0e74c --- /dev/null +++ b/grafana/grafana_dashboards_backup/a0defb5e-33db-41da-ba04-17d698577b72.json @@ -0,0 +1,632 @@ +{ + "meta": { + "type": "db", + "canSave": true, + "canEdit": true, + "canAdmin": true, + "canStar": true, + "canDelete": true, + "slug": "gnosis-p2p-network-stats", + "url": "/d/a0defb5e-33db-41da-ba04-17d698577b72/gnosis-p2p-network-stats", + "expires": "0001-01-01T00:00:00Z", + "created": "2024-10-04T15:27:26Z", + "updated": "2024-10-21T13:02:33Z", + "updatedBy": "gnosisid_hugo.serodio@gnosis.io", + "createdBy": "gnosisid_hugo.serodio@gnosis.io", + "version": 24, + "hasAcl": false, + "isFolder": false, + "folderId": 2, + "folderUid": "c049b211-0058-4301-b9e2-aa54973022e1", + "folderTitle": "Cerebro Dashboards", + "folderUrl": "/dashboards/f/c049b211-0058-4301-b9e2-aa54973022e1/cerebro-dashboards", + "provisioned": false, + "provisionedExternalId": "", + "annotationsPermissions": { + "dashboard": { + "canAdd": true, + "canEdit": true, + "canDelete": true + }, + "organization": { + "canAdd": true, + "canEdit": true, + "canDelete": true + } + }, + "publicDashboardAccessToken": "", + "publicDashboardUid": "", + "publicDashboardEnabled": false + }, + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n \"Sample\n
", + "mode": "html" + }, + "pluginVersion": "10.1.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 16, + "x": 8, + "y": 0 + }, + "id": 6, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n

P2P Network

\n
", + "mode": "markdown" + }, + "pluginVersion": "10.1.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 4, + "w": 16, + "x": 0, + "y": 2 + }, + "id": 8, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n \n

Dashboard Overview

\n \n

This dashboard provides a comprehensive view of Peer-to-Peer (P2P) network data. Key metrics include the number of active peers, geo location, client diversity and more.

\n
\n", + "mode": "html" + }, + "pluginVersion": "10.1.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#3e6957", + "value": null + }, + { + "color": "#3e6957", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 2 + }, + "id": 4, + "options": { + "colorMode": "background_solid", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT * FROM dbt.p2p_valtrack_nodes_cnt_1d\nWHERE \n active_nodes>0\nORDER BY date DESC\nLIMIT 1\n", + "refId": "A" + } + ], + "title": "Active Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#3e6957", + "value": null + }, + { + "color": "#3e6957", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 2 + }, + "id": 5, + "options": { + "colorMode": "background_solid", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT COUNT(*) FROM dbt.p2p_valtrack_nodes_cnt_country_1d\nWHERE \n active_nodes>0\nGROUP BY date\nORDER BY date DESC\nLIMIT 1", + "refId": "A" + } + ], + "title": "Countries", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 13, + "x": 0, + "y": 6 + }, + "id": 2, + "options": { + "basemap": { + "config": { + "showLabels": true, + "theme": "auto" + }, + "name": "Layer 0", + "type": "carto" + }, + "controls": { + "mouseWheelZoom": false, + "showAttribution": true, + "showDebug": false, + "showMeasure": false, + "showScale": false, + "showZoom": false + }, + "layers": [ + { + "config": { + "blur": 8, + "radius": 4, + "weight": { + "fixed": 1, + "max": 1, + "min": 0 + } + }, + "name": "Layer 1", + "opacity": 1, + "tooltip": true, + "type": "heatmap" + } + ], + "tooltip": { + "mode": "details" + }, + "view": { + "allLayers": true, + "id": "zero", + "lat": 0, + "lon": 0, + "zoom": 1 + } + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT \n latitude\n ,longitude\n ,COUNT(*) as cnt\nFROM \n dbt.p2p_valtrack_nodes_geo_last_day \nGROUP BY 1,2", + "refId": "A" + } + ], + "type": "geomap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 6 + }, + "id": 7, + "options": { + "baidu": { + "callback": "bmapReady", + "key": "" + }, + "editor": { + "format": "auto" + }, + "editorMode": "code", + "gaode": { + "key": "", + "plugin": "AMap.Scale,AMap.ToolBar" + }, + "getOption": "const series = context.panel.data.series.map((s) => {\n const sData = s.fields.find((f) => f.type === 'number').values.buffer || s.fields.find((f) => f.type === 'number').values;\n const sTime = s.fields.find((f) => f.type === 'time').values.buffer || s.fields.find((f) => f.type === 'time').values;\n\n return {\n name: \"nodes\",\n type: 'line',\n showSymbol: false,\n areaStyle: {\n opacity: 0.1,\n },\n lineStyle: {\n width: 1,\n },\n data: sData.map((d, i) => [sTime[i], d.toFixed(2)]),\n };\n});\n\n/**\n * Enable Data Zoom by default\n */\nsetTimeout(() => context.panel.chart.dispatchAction({\n type: 'takeGlobalCursor',\n key: 'dataZoomSelect',\n dataZoomSelectActive: true,\n}), 500);\n\n/**\n * Update Time Range on Zoom\n */\ncontext.panel.chart.on('datazoom', function (params) {\n const startValue = params.batch[0]?.startValue;\n const endValue = params.batch[0]?.endValue;\n locationService.partial({ from: startValue, to: endValue });\n});\n\nreturn {\n backgroundColor: 'transparent',\n tooltip: {\n trigger: 'axis',\n },\n toolbox: {\n feature: {\n dataZoom: {\n yAxisIndex: 'none',\n icon: {\n zoom: 'path://',\n back: 'path://',\n },\n },\n saveAsImage: {},\n }\n },\n xAxis: {\n type: 'time',\n name: 'Hour',\n nameLocation: 'middle',\n nameGap: 20\n },\n yAxis: {\n type: 'value',\n min: 'dataMin',\n name: 'Count',\n nameLocation: 'middle',\n nameRotation: 90,\n nameGap: 50\n },\n grid: {\n left: '2%',\n right: '2%',\n top: '2%',\n bottom: 24,\n containLabel: true,\n },\n series,\n};", + "google": { + "callback": "gmapReady", + "key": "" + }, + "map": "none", + "renderer": "canvas", + "themeEditor": { + "config": "{}", + "name": "default" + }, + "visualEditor": { + "code": "return {\n dataset: context.editor.dataset,\n series: context.editor.series,\n xAxis: {\n type: 'time',\n },\n yAxis: {\n type: 'value',\n min: 'dataMin',\n },\n}\n", + "dataset": [], + "series": [] + } + }, + "pluginVersion": "6.4.1", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT * FROM dbt.p2p_valtrack_nodes_cnt_1h\nWHERE\n DATE_TRUNC('day',date) = DATE_TRUNC('day',now() - INTERVAL 15 DAY)", + "refId": "A" + } + ], + "title": "Last day seen nodes", + "type": "volkovlabs-echarts-panel" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "gridPos": { + "h": 13, + "w": 21, + "x": 0, + "y": 18 + }, + "id": 9, + "options": { + "baidu": { + "callback": "bmapReady", + "key": "" + }, + "editor": { + "format": "auto" + }, + "editorMode": "code", + "gaode": { + "key": "", + "plugin": "AMap.Scale,AMap.ToolBar" + }, + "getOption": "// 1. **Define the Base and Extra Color Palettes**\nconst baseColorPalette = [\n 'rgba(118, 150, 137, 0.7)', // '#769689' - Light Green / Forest Light (keep)\n 'rgba(221, 113, 67, 0.7)', // '#dd7143' - Orange (keep)\n];\n\nconst extraColors = [\n 'rgba(84, 112, 198, 0.7)', // '#5470C6'\n 'rgba(145, 204, 117, 0.7)', // '#91CC75'\n 'rgba(250, 200, 88, 0.7)', // '#FAC858'\n 'rgba(238, 102, 102, 0.7)', // '#EE6666'\n 'rgba(115, 192, 222, 0.7)', // '#73C0DE'\n 'rgba(59, 162, 114, 0.7)', // '#3BA272'\n 'rgba(154, 96, 180, 0.7)', // '#9A60B4'\n 'rgba(234, 124, 204, 0.7)', // '#EA7CCC'\n 'rgba(255, 159, 127, 0.7)', // '#FF9F7F'\n 'rgba(231, 188, 243, 0.7)', // '#E7BCF3'\n 'rgba(212, 130, 101, 0.7)', // '#D48265'\n 'rgba(145, 199, 174, 0.7)', // '#91C7AE'\n 'rgba(116, 159, 131, 0.7)', // '#749F83'\n 'rgba(202, 130, 34, 0.7)', // '#CA8622'\n 'rgba(189, 162, 154, 0.7)', // '#BDA29A'\n];\n\n// 2. **Generate a Dynamic Color Palette**\nfunction generateColorPalette(count) {\n const fullPalette = [...baseColorPalette, ...extraColors];\n\n // If more colors are needed, cycle through the extraColors\n while (fullPalette.length < count) {\n fullPalette.push(...extraColors);\n }\n\n return fullPalette.slice(0, count); // Return only as many as needed\n}\n\n// 3. **Generate the Series Data with Safe Field Access**\nconst seriesData = context.panel.data.series.map((s, seriesIndex) => {\n // Check if s.fields exists and is an array\n if (!s.fields || !Array.isArray(s.fields)) {\n console.warn(`Series at index ${seriesIndex} is missing 'fields'. Skipping.`);\n return [];\n }\n\n // Find required fields\n const cntField = s.fields.find((f) => f.type === 'number');\n const datetimeField = s.fields.find((f) => f.name === 'datetime_label');\n const itemField = s.fields.find((f) => f.name === 'item');\n\n // If any required field is missing, skip this series\n if (!cntField || !datetimeField || !itemField) {\n console.warn(`Series at index ${seriesIndex} is missing one or more required fields. Skipping.`);\n return [];\n }\n\n // Extract values, handling buffer if present\n const cntData = cntField.values.buffer || cntField.values;\n const datetimeLabels = datetimeField.values.buffer || datetimeField.values;\n const items = itemField.values.buffer || itemField.values;\n\n // Check that all arrays have the same length\n const length = Math.min(cntData.length, datetimeLabels.length, items.length);\n\n if (cntData.length !== datetimeLabels.length || cntData.length !== items.length) {\n console.warn(`Series at index ${seriesIndex} has mismatched field lengths.`);\n }\n\n // Map the data\n return Array.from({ length }, (_, i) => ({\n datetime: datetimeLabels[i],\n item: items[i],\n cnt: parseFloat(cntData[i]).toFixed(2),\n }));\n}).flat();\n\n// 4. **Get Unique Items for Separate Series**\nconst uniqueItems = [...new Set(seriesData.map((d) => d.item))];\n\n// 5. **Generate a Dynamic Color Palette Based on the Number of Unique Items**\nconst colorPalette = generateColorPalette(uniqueItems.length);\n\n// 6. **Get Unique Datetime Labels for the X-Axis**\nconst uniqueSlots = Array.from(new Set(seriesData.map((d) => d.datetime)))\n .sort((a, b) => new Date(a) - new Date(b)); // Sort chronologically\n\n// 7. **Create a Mapping from Item to Datetime to Count**\nconst dataMap = uniqueItems.reduce((acc, item) => {\n acc[item] = new Map();\n uniqueSlots.forEach(slot => {\n acc[item].set(slot, 0); // Initialize with 0\n });\n return acc;\n}, {});\n\nseriesData.forEach(d => {\n if (dataMap[d.item]) {\n dataMap[d.item].set(d.datetime, parseFloat(d.cnt));\n }\n});\n\n// 8. **Create a Series for Each Unique Item**\nconst series = uniqueItems.map((item, index) => {\n const itemData = uniqueSlots.map(slot => dataMap[item].get(slot) || 0);\n\n return {\n name: item,\n type: 'bar',\n stack: 'total',\n data: itemData,\n itemStyle: {\n color: colorPalette[index % colorPalette.length], // Assign color cyclically with opacity\n borderColor: 'rgba(26, 26, 26, 1)', // Dark Grey border with full opacity\n borderWidth: 0.3, // Set border width for each bar\n },\n emphasis: { // Highlight on hover\n itemStyle: {\n borderColor: '#000', // Highlight border color (Black)\n borderWidth: 2, // Highlight border width\n },\n },\n };\n});\n\n// 9. **Enable Data Zoom by Default**\nsetTimeout(() => {\n context.panel.chart.dispatchAction({\n type: 'takeGlobalCursor',\n key: 'dataZoomSelect',\n dataZoomSelectActive: true,\n });\n}, 500);\n\n// 10. **Update Time Range on Zoom**\ncontext.panel.chart.on('datazoom', function (params) {\n const startValue = params.batch[0]?.startValue;\n const endValue = params.batch[0]?.endValue;\n locationService.partial({ from: startValue, to: endValue });\n});\n\n// 11. **Configure ECharts Options with Enhanced Styling**\nconst option = {\n backgroundColor: '#fffcfa', // Light Cream Background\n\n // **Title Component**\n title: {\n text: 'Daily Nodes seen in the Network', // Main title text\n subtext: 'per Countries Cohorts', // Optional subtitle text\n left: 'center', // Position the title centrally\n top: '2%', // Position the title slightly below the top edge\n textStyle: {\n fontSize: 18, // Main title font size\n fontWeight: 'bold', // Main title font weight\n color: '#231e10', // Main title color (Black)\n },\n subtextStyle: {\n fontSize: 14, // Subtitle font size\n color: '#3e6957', // Subtitle color (Forest)\n },\n },\n\n // **Tooltip Configuration**\n tooltip: {\n trigger: 'axis',\n axisPointer: { type: 'shadow' },\n formatter: function (params) {\n // Determine column count dynamically based on number of series\n const columnCount = params.length > 6 ? 3 : 2;\n\n // Start with the axis value (e.g., category name)\n let tooltipText = `\n
\n ${params[0].axisValue}
\n
\n `;\n\n // Iterate through each param to build tooltip entries\n params.forEach(param => {\n let valueAsInt = parseInt(param.value, 10);\n tooltipText += `\n
\n ${param.marker} ${param.seriesName}: ${valueAsInt}\n
\n `;\n });\n\n // Close the divs\n tooltipText += `\n
\n
\n `;\n return tooltipText;\n },\n },\n\n // **Legend Configuration**\n legend: {\n data: uniqueItems, // Array of unique item names\n top: '18%', // Position the legend below the title\n left: 'center', // Center the legend horizontally\n orient: 'horizontal', // Arrange legend items horizontally\n type: 'scroll', // Enable scrolling when items exceed the container\n pageIconSize: 8, // Reduce the size of scroll buttons to make scrollbar less high\n pageIconInactiveColor: '#ccc', // Inactive scroll button color\n pageIconActiveColor: '#333', // Active scroll button color\n pageTextStyle: {\n color: '#333',\n fontSize: 10, // Smaller font size for page indicators\n },\n width: '60%', // Limit legend width to allow wrapping into multiple lines\n textStyle: {\n fontSize: 12, // Legend text size\n color: '#231e10', // Legend text color (Black)\n },\n itemGap: 15, // Space between legend items\n selectedMode: 'multiple', // Allow multiple series to be toggled\n },\n\n // **X-Axis Configuration**\n xAxis: {\n type: 'category',\n name: 'Date',\n data: uniqueSlots,\n boundaryGap: true, // Ensure bars are nicely spaced\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize x-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n //axisLabel: {\n // rotate: 45, // Rotate labels for better readability\n // color: '#231e10', // X-axis label color (Black)\n // fontSize: 12, // X-axis label font size\n // },\n },\n\n // **Y-Axis Configuration**\n yAxis: {\n type: 'value',\n name: 'Count',\n axisLabel: {\n formatter: '{value}',\n color: '#231e10', // Y-axis label color (Black)\n fontSize: 12, // Y-axis label font size\n },\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize y-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n splitLine: {\n lineStyle: {\n type: 'dashed',\n color: '#231e10', // Changed from '#cbc3ad' to '#231e10' for better visibility\n width: 1, // Slightly thicker dashed lines\n },\n },\n },\n\n // **Grid Configuration**\n grid: {\n left: '2%',\n right: '10%',\n top: '25%', // Increased top margin to accommodate legend and title\n bottom: '15%', // Reduced bottom margin to bring the slider closer to the plot\n containLabel: true,\n backgroundColor: '#ffffff', // Plot area's background color (White)\n borderColor: '#cbc3ad', // Border color of the plot area (Cream Dark)\n borderWidth: 1, // Border width of the plot area\n borderRadius: 5, // Rounded corners for the plot area\n shadowColor: 'rgba(0, 0, 0, 0.05)', // Subtle shadow for depth\n shadowBlur: 10,\n shadowOffsetX: 0,\n shadowOffsetY: 2,\n },\n\n // **Data Zoom Configuration**\n dataZoom: [\n {\n type: 'inside', // Enable zooming inside the chart\n start: 0,\n end: 100,\n },\n {\n type: 'slider', // Enable a slider for zooming\n start: 0,\n end: 100,\n height: 10, // Reduce the height of the data zoom slider\n bottom: '10%', // Position it closer to the plot\n handleIcon: 'M10.7,11.9v-1.3H9.3v1.3c-4.9,0.3-8.8,4.4-8.8,9.4c0,5,3.9,9.1,8.8,9.4v1.3h1.4v-1.3c4.9-0.3,8.8-4.4,8.8-9.4C19.5,16.3,15.6,12.2,10.7,11.9z', // Customize handle icon\n handleSize: '80%', // Adjust handle size\n handleStyle: {\n color: '#6f592c', // Handle color (Saddle Brown)\n },\n textStyle: {\n color: '#231e10', // Text color for data zoom\n fontSize: 10, // Text size\n },\n backgroundColor: 'rgba(228, 221, 203, 1)', // '#e4ddcb' - Cream Medium with full opacity\n fillerColor: 'rgba(118, 150, 137, 0.7)', // '#769689' - Light Green with opacity\n borderColor: 'rgba(203, 195, 173, 1)', // '#cbc3ad' - Cream Dark with full opacity\n },\n ],\n\n // **Assign Series Data**\n series: series,\n};\n\n// 12. **Return the Configured Option**\nreturn option;\n", + "google": { + "callback": "gmapReady", + "key": "" + }, + "map": "none", + "renderer": "canvas", + "themeEditor": { + "config": "{}", + "name": "default" + }, + "visualEditor": { + "code": "return {\n dataset: context.editor.dataset,\n series: context.editor.series,\n xAxis: {\n type: 'time',\n },\n yAxis: {\n type: 'value',\n min: 'dataMin',\n },\n}\n", + "dataset": [], + "series": [] + } + }, + "pluginVersion": "6.4.1", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT \ndate as datetime\n,CAST(date AS text) AS datetime_label\n,country AS item\n,active_nodes AS cnt\n FROM dbt.p2p_valtrack_nodes_cnt_country_1d\n ORDER BY date, cnt DESC", + "refId": "A" + } + ], + "type": "volkovlabs-echarts-panel" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "gridPos": { + "h": 10, + "w": 14, + "x": 0, + "y": 31 + }, + "id": 1, + "options": { + "baidu": { + "callback": "bmapReady", + "key": "" + }, + "editor": { + "format": "auto" + }, + "editorMode": "code", + "gaode": { + "key": "", + "plugin": "AMap.Scale,AMap.ToolBar" + }, + "getOption": "const series = context.panel.data.series.map((s) => {\n const sData = s.fields.find((f) => f.type === 'number').values.buffer || s.fields.find((f) => f.type === 'number').values;\n const sTime = s.fields.find((f) => f.type === 'time').values.buffer || s.fields.find((f) => f.type === 'time').values;\n\n return {\n name: \"nodes\",\n type: 'bar',\n showSymbol: false,\n areaStyle: {\n opacity: 0.1,\n },\n lineStyle: {\n width: 1,\n },\n itemStyle: { // Set the bar color\n color: 'rgba(118, 150, 137, 0.7)', // Light Green / Forest Light\n borderColor: 'rgba(26, 26, 26, 1)', // Optional: Dark Grey border with full opacity\n borderWidth: 0.3, // Optional: Set border width for each bar\n },\n data: sData.map((d, i) => [sTime[i], d.toFixed(2)]),\n };\n});\n\n/**\n * Enable Data Zoom by default\n */\nsetTimeout(() => context.panel.chart.dispatchAction({\n type: 'takeGlobalCursor',\n key: 'dataZoomSelect',\n dataZoomSelectActive: true,\n}), 500);\n\n/**\n * Update Time Range on Zoom\n */\ncontext.panel.chart.on('datazoom', function (params) {\n const startValue = params.batch[0]?.startValue;\n const endValue = params.batch[0]?.endValue;\n locationService.partial({ from: startValue, to: endValue });\n});\n\nreturn {\n backgroundColor: '#fffcfa',\n\n // **Title Component**\n title: {\n text: 'Daily Nodes seen in the Network', // Main title text\n // subtext: 'Inclusion Distance Cohorts', // Optional subtitle text\n left: 'center', // Position the title centrally\n top: '2%', // Position the title slightly below the top edge\n textStyle: {\n fontSize: 18, // Main title font size\n fontWeight: 'bold', // Main title font weight\n color: '#231e10', // Main title color (Black)\n },\n subtextStyle: {\n fontSize: 14, // Subtitle font size\n color: '#3e6957', // Subtitle color (Forest)\n },\n },\n\n tooltip: {\n trigger: 'axis',\n },\n toolbox: {\n feature: {\n dataZoom: {\n yAxisIndex: 'none',\n icon: {\n zoom: 'path://',\n back: 'path://',\n },\n },\n saveAsImage: {},\n }\n },\n xAxis: {\n type: 'time',\n name: 'Day',\n nameLocation: 'middle',\n nameGap: 20,\n boundaryGap: true,\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize x-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n },\n\n yAxis: {\n type: 'value',\n min: 'dataMin',\n name: 'Count',\n nameLocation: 'middle',\n nameRotation: 90,\n nameGap: 50,\n axisLabel: {\n formatter: '{value}',\n color: '#231e10', // Y-axis label color (Black)\n fontSize: 12, // Y-axis label font size\n },\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize y-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n splitLine: {\n lineStyle: {\n type: 'dashed',\n color: '#231e10', // Changed from '#cbc3ad' to '#231e10' for better visibility\n width: 1, // Slightly thicker dashed lines\n },\n },\n },\n\n dataZoom: [\n {\n type: 'inside', // Enable zooming inside the chart\n start: 0,\n end: 100,\n },\n {\n type: 'slider', // Enable a slider for zooming\n start: 0,\n end: 100,\n height: 10, // Reduce the height of the data zoom slider\n bottom: '10%', // Position it closer to the plot\n handleIcon: 'M10.7,11.9v-1.3H9.3v1.3c-4.9,0.3-8.8,4.4-8.8,9.4c0,5,3.9,9.1,8.8,9.4v1.3h1.4v-1.3c4.9-0.3,8.8-4.4,8.8-9.4C19.5,16.3,15.6,12.2,10.7,11.9z', // Customize handle icon\n handleSize: '80%', // Adjust handle size\n handleStyle: {\n color: '#6f592c', // Handle color (Saddle Brown)\n },\n textStyle: {\n color: '#231e10', // Text color for data zoom\n fontSize: 10, // Text size\n },\n backgroundColor: 'rgba(228, 221, 203, 1)', // '#e4ddcb' - Cream Medium with full opacity\n fillerColor: 'rgba(118, 150, 137, 0.7)', // '#769689' - Light Green with opacity\n borderColor: 'rgba(203, 195, 173, 1)', // '#cbc3ad' - Cream Dark with full opacity\n },\n ],\n\n grid: {\n left: '2%',\n right: '10%',\n top: '25%', // Increased top margin to accommodate legend and title\n bottom: '15%', // Reduced bottom margin to bring the slider closer to the plot\n containLabel: true,\n backgroundColor: '#ffffff', // Plot area's background color (White)\n borderColor: '#cbc3ad', // Border color of the plot area (Cream Dark)\n borderWidth: 1, // Border width of the plot area\n borderRadius: 5, // Rounded corners for the plot area\n shadowColor: 'rgba(0, 0, 0, 0.05)', // Subtle shadow for depth\n shadowBlur: 10,\n shadowOffsetX: 0,\n shadowOffsetY: 2,\n },\n series,\n};\n", + "google": { + "callback": "gmapReady", + "key": "" + }, + "map": "none", + "renderer": "canvas", + "themeEditor": { + "config": "{}", + "name": "default" + }, + "visualEditor": { + "code": "return {\n dataset: context.editor.dataset,\n series: context.editor.series,\n xAxis: {\n type: 'time',\n },\n yAxis: {\n type: 'value',\n min: 'dataMin',\n },\n}\n", + "dataset": [], + "series": [] + } + }, + "pluginVersion": "6.4.1", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT * FROM dbt.p2p_valtrack_nodes_cnt_1d", + "refId": "A" + } + ], + "type": "volkovlabs-echarts-panel" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Gnosis P2P Network Stats", + "uid": "a0defb5e-33db-41da-ba04-17d698577b72", + "version": 24, + "weekStart": "" + } +} diff --git a/grafana/grafana_dashboards_backup/b0e1d65b-c5d9-4f58-9e2b-9c29779c2c53.json b/grafana/grafana_dashboards_backup/b0e1d65b-c5d9-4f58-9e2b-9c29779c2c53.json new file mode 100644 index 0000000..99f22cd --- /dev/null +++ b/grafana/grafana_dashboards_backup/b0e1d65b-c5d9-4f58-9e2b-9c29779c2c53.json @@ -0,0 +1,1168 @@ +{ + "meta": { + "type": "db", + "canSave": true, + "canEdit": true, + "canAdmin": true, + "canStar": true, + "canDelete": true, + "slug": "consensus-overview", + "url": "/d/b0e1d65b-c5d9-4f58-9e2b-9c29779c2c53/consensus-overview", + "expires": "0001-01-01T00:00:00Z", + "created": "2024-10-18T12:26:17Z", + "updated": "2024-10-23T06:48:40Z", + "updatedBy": "gnosisid_hugo.serodio@gnosis.io", + "createdBy": "gnosisid_hugo.serodio@gnosis.io", + "version": 28, + "hasAcl": false, + "isFolder": false, + "folderId": 2, + "folderUid": "c049b211-0058-4301-b9e2-aa54973022e1", + "folderTitle": "Cerebro Dashboards", + "folderUrl": "/dashboards/f/c049b211-0058-4301-b9e2-aa54973022e1/cerebro-dashboards", + "provisioned": false, + "provisionedExternalId": "", + "annotationsPermissions": { + "dashboard": { + "canAdd": true, + "canEdit": true, + "canDelete": true + }, + "organization": { + "canAdd": true, + "canEdit": true, + "canDelete": true + } + }, + "publicDashboardAccessToken": "", + "publicDashboardUid": "", + "publicDashboardEnabled": false + }, + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n \"Sample\n
", + "mode": "html" + }, + "pluginVersion": "10.1.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 16, + "x": 8, + "y": 0 + }, + "id": 3, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n

Consensus Overview

\n
", + "mode": "markdown" + }, + "pluginVersion": "10.1.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 15, + "w": 10, + "x": 0, + "y": 2 + }, + "id": 27, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n \n

Blocks Production

\n \n

This dashboard provides a comprehensive view of Peer-to-Peer (P2P) network data. Key metrics include the number of active peers, geo location, client diversity and more.

\n
\n", + "mode": "html" + }, + "pluginVersion": "10.1.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#3e6957", + "value": null + }, + { + "color": "#fffcfc", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 10, + "y": 2 + }, + "id": 8, + "options": { + "colorMode": "background_solid", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT\nactive\nFROM dbt.consensus_validators_participation_rate\nORDER BY day DESC\nLIMIT 1", + "refId": "A" + } + ], + "title": "Last Day Active Validators", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#3e6957", + "value": null + }, + { + "color": "#fffcfc", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 15, + "y": 2 + }, + "id": 28, + "options": { + "colorMode": "background_solid", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT\nactive\nFROM dbt.consensus_validators_participation_rate\nORDER BY day DESC\nLIMIT 1", + "refId": "A" + } + ], + "title": "Last Day Active Validators", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#3e6957", + "value": null + }, + { + "color": "#fffcfc", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 2 + }, + "id": 29, + "options": { + "colorMode": "background_solid", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT\nactive\nFROM dbt.consensus_validators_participation_rate\nORDER BY day DESC\nLIMIT 1", + "refId": "A" + } + ], + "title": "Last Day Active Validators", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "gridPos": { + "h": 12, + "w": 14, + "x": 10, + "y": 5 + }, + "id": 6, + "options": { + "baidu": { + "callback": "bmapReady", + "key": "" + }, + "editor": { + "format": "none" + }, + "editorMode": "code", + "gaode": { + "key": "", + "plugin": "AMap.Scale,AMap.ToolBar" + }, + "getOption": "const series = [];\n\n// Define your custom color palette with opacity (70%)\nconst colorPalette = [\n 'rgba(118, 150, 137, 0.7)', // '#769689' - Light Green / Forest Light\n 'rgba(221, 113, 67, 0.7)', // '#dd7143' - Orange\n 'rgba(154, 96, 180, 0.7)', // '#9A60B4' - Purple\n];\n\n// Extract unique datetime_labels for the x-axis first\nconst uniqueSlots = Array.from(new Set(\n context.panel.data.series.flatMap(s => {\n const timeField = s.fields.find(f => f.name === 'datetime_label');\n return timeField.values.buffer || timeField.values;\n })\n)).sort((a, b) => new Date(a) - new Date(b)); // Assuming 'day' is a date\n\n// Group the data by 'label'\nconst groupedData = context.panel.data.series.reduce((acc, s) => {\n const labelField = s.fields.find((f) => f.name === 'label');\n const cntField = s.fields.find((f) => f.name === 'cnt');\n const timeField = s.fields.find((f) => f.name === 'datetime_label');\n\n if (!labelField || !cntField || !timeField) {\n // Handle missing fields gracefully\n return acc;\n }\n\n // Extract values, handling buffer if present\n const cohorts = labelField.values.buffer || labelField.values;\n const cntData = cntField.values.buffer || cntField.values;\n const timeData = timeField.values.buffer || timeField.values;\n\n cohorts.forEach((cohortValue, index) => {\n if (!acc[cohortValue]) {\n acc[cohortValue] = Array(uniqueSlots.length).fill(0);\n }\n const slotIndex = uniqueSlots.indexOf(timeData[index]);\n if (slotIndex !== -1) {\n acc[cohortValue][slotIndex] += parseFloat(cntData[index]) || 0;\n }\n });\n\n return acc;\n}, {});\n\n// Generate a bar series for each cohort with color assignment\nObject.keys(groupedData).forEach((cohortValue, index) => {\n series.push({\n name: `${cohortValue}`, // Label each series as \"Cohort X\"\n type: 'bar',\n stack: 'stack', // Enable stacking\n data: groupedData[cohortValue].map(y => parseFloat(y).toFixed(2)),\n itemStyle: {\n color: colorPalette[index % colorPalette.length], // Assign color cyclically with opacity\n borderColor: 'rgba(26, 26, 26, 1)', // Dark Grey border with full opacity\n borderWidth: 0.3, // Set border width for each bar\n },\n emphasis: { // Highlight on hover\n itemStyle: {\n borderColor: '#000', // Highlight border color (Black)\n borderWidth: 2, // Highlight border width\n },\n },\n });\n});\n\n// Deduplicate total data\nconst totalDataMap = new Map();\n\ncontext.panel.data.series.forEach(s => {\n const timeField = s.fields.find(f => f.name === 'datetime_label');\n const totalField = s.fields.find(f => f.name === 'total');\n \n if (timeField && totalField) {\n const timeData = timeField.values.buffer || timeField.values;\n const totalValues = totalField.values.buffer || totalField.values;\n \n timeData.forEach((time, idx) => {\n if (!totalDataMap.has(time)) {\n totalDataMap.set(time, parseFloat(totalValues[idx]) || 0);\n }\n // If 'total' appears in multiple series, ensure it remains consistent\n // Optionally, you can verify consistency here\n });\n }\n});\n\n// Align the total data with uniqueSlots\nconst totalData = uniqueSlots.map(slot => totalDataMap.get(slot) || 0);\n\n// Add the 'Total' line series\nseries.push({\n name: 'Total',\n type: 'line',\n yAxisIndex: 1, // Assign to the second y-axis\n data: totalData.map(value => parseFloat(value).toFixed(2)), // Ensure numerical values\n smooth: true, // Makes the line smooth\n lineStyle: {\n color: '#133629',\n width: 2,\n },\n itemStyle: {\n color: '#133629',\n },\n tooltip: {\n valueFormatter: value => parseInt(value, 10),\n },\n});\n\n/**\n * Enable Data Zoom by default\n */\nsetTimeout(() => {\n context.panel.chart.dispatchAction({\n type: 'takeGlobalCursor',\n key: 'dataZoomSelect',\n dataZoomSelectActive: true,\n });\n}, 500);\n\n/**\n * Update Time Range on Zoom\n */\ncontext.panel.chart.on('datazoom', function (params) {\n const startValue = params.batch[0]?.startValue;\n const endValue = params.batch[0]?.endValue;\n locationService.partial({ from: startValue, to: endValue });\n});\n\n// Configure ECharts options with Title, Legend, Tooltip, etc.\nconst option = {\n backgroundColor: '#fffcfa', // Light Cream Background\n\n // Add the Title Component\n title: {\n text: 'Active Validators', // Main title text\n subtext: 'Status Cohorts', // Optional subtitle text\n left: 'center', // Position the title centrally\n top: '2%', // Position the title slightly below the top edge\n textStyle: {\n fontSize: 18, // Main title font size\n fontWeight: 'bold', // Main title font weight\n color: '#231e10', // Main title color (Black)\n },\n subtextStyle: {\n fontSize: 14, // Subtitle font size\n color: '#3e6957', // Subtitle color (Forest)\n },\n },\n\n tooltip: {\n trigger: 'axis',\n axisPointer: { type: 'shadow' },\n formatter: function (params) {\n // Determine column count dynamically based on number of series\n const columnCount = params.length > 6 ? 3 : 2;\n\n // Start with the axis value (e.g., category name)\n let tooltipText = `\n
\n ${params[0].axisValue}
\n
\n `;\n\n // Iterate through each param to build tooltip entries\n params.forEach(param => {\n let valueAsInt = parseInt(param.value, 10);\n tooltipText += `\n
\n ${param.marker} ${param.seriesName}: ${valueAsInt}\n
\n `;\n });\n\n // Close the divs\n tooltipText += `\n
\n
\n `;\n return tooltipText;\n },\n },\n\n legend: {\n data: series.map((s) => s.name),\n top: '18%', // Position the legend below the title\n left: 'center', // Center the legend horizontally\n orient: 'horizontal', // Arrange legend items horizontally\n type: 'scroll', // Enable scrolling when items exceed the container\n pageIconSize: 8, // Reduce the size of scroll buttons to make scrollbar less high\n pageIconInactiveColor: '#ccc', // Inactive scroll button color\n pageIconActiveColor: '#333', // Active scroll button color\n pageTextStyle: {\n color: '#333',\n fontSize: 10, // Smaller font size for page indicators\n },\n width: '60%', // Limit legend width to allow wrapping into multiple lines\n textStyle: {\n fontSize: 12, // Legend text size\n color: '#231e10', // Legend text color (Black)\n },\n itemGap: 15, // Space between legend items\n selectedMode: 'multiple', // Allow multiple series to be toggled\n },\n\n xAxis: {\n type: 'category',\n data: uniqueSlots,\n boundaryGap: true, // Ensure bars are nicely spaced\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize x-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n },\n\n yAxis: [\n {\n // First Y-Axis (Left) for Bar Charts\n type: 'value',\n name: 'Count',\n axisLabel: {\n formatter: '{value}',\n color: '#231e10', // Black\n fontSize: 12,\n },\n axisLine: {\n lineStyle: {\n color: '#231e10',\n width: 2,\n },\n },\n splitLine: {\n lineStyle: {\n type: 'dashed',\n color: '#231e10',\n width: 1,\n },\n },\n },\n {\n // Second Y-Axis (Right) for Total Line\n type: 'value',\n name: 'Total',\n position: 'right',\n axisLabel: {\n formatter: '{value}',\n color: '#133629',\n fontSize: 12,\n },\n axisLine: {\n lineStyle: {\n color: '#133629',\n width: 2,\n },\n },\n splitLine: {\n show: false, // Optionally hide split lines for clarity\n },\n },\n ],\n\n series: series,\n\n dataZoom: [\n {\n type: 'inside', // Enable zooming inside the chart\n start: 80,\n end: 100,\n },\n {\n type: 'slider', // Enable a slider for zooming\n start: 0,\n end: 100,\n height: 10, // Reduce the height of the data zoom slider\n bottom: '10%', // Position it closer to the plot\n handleIcon: 'M10.7,11.9v-1.3H9.3v1.3c-4.9,0.3-8.8,4.4-8.8,9.4c0,5,3.9,9.1,8.8,9.4v1.3h1.4v-1.3c4.9-0.3,8.8-4.4,8.8-9.4C19.5,16.3,15.6,12.2,10.7,11.9z', // Customize handle icon\n handleSize: '80%', // Adjust handle size\n handleStyle: {\n color: '#6f592c', // Handle color (Saddle Brown)\n },\n textStyle: {\n color: '#231e10', // Text color for data zoom\n fontSize: 10, // Text size\n },\n backgroundColor: 'rgba(228, 221, 203, 1)', // '#e4ddcb' - Cream Medium with full opacity\n fillerColor: 'rgba(118, 150, 137, 0.7)', // '#769689' - Light Green with opacity\n borderColor: 'rgba(203, 195, 173, 1)', // '#cbc3ad' - Cream Dark with full opacity\n },\n ],\n\n grid: {\n left: '2%',\n right: '10%',\n top: '25%', // Increased top margin to accommodate legend and title\n bottom: '15%', // Reduced bottom margin to bring the slider closer to the plot\n containLabel: true,\n backgroundColor: '#ffffff', // Plot area's background color (White)\n borderColor: '#cbc3ad', // Border color of the plot area (Cream Dark)\n borderWidth: 1, // Border width of the plot area\n borderRadius: 5, // Rounded corners for the plot area\n shadowColor: 'rgba(0, 0, 0, 0.05)', // Subtle shadow for depth\n shadowBlur: 10,\n shadowOffsetX: 0,\n shadowOffsetY: 2,\n },\n};\n\nreturn option;\n", + "google": { + "callback": "gmapReady", + "key": "" + }, + "map": "none", + "renderer": "canvas", + "themeEditor": { + "config": "{}", + "name": "custom" + }, + "visualEditor": { + "code": "return {\n dataset: context.editor.dataset,\n series: context.editor.series,\n xAxis: {\n type: 'time',\n },\n yAxis: {\n type: 'value',\n min: 'dataMin',\n },\n}\n", + "dataset": [ + { + "name": "f_inclusion_slot", + "source": "A" + }, + { + "name": "cnt", + "source": "A" + } + ], + "series": [] + } + }, + "pluginVersion": "6.4.1", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT\n*\n,CAST(day AS text) AS datetime_label\n,SUM(cnt) OVER(ORDER BY day) AS total\nFROM dbt.consensus_validators_activations\nORDER BY day ASC", + "refId": "A" + } + ], + "type": "volkovlabs-echarts-panel" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 20, + "x": 2, + "y": 17 + }, + "id": 22, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n
", + "mode": "markdown" + }, + "pluginVersion": "10.1.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "gridPos": { + "h": 12, + "w": 14, + "x": 0, + "y": 18 + }, + "id": 5, + "options": { + "baidu": { + "callback": "bmapReady", + "key": "" + }, + "editor": { + "format": "none" + }, + "editorMode": "code", + "gaode": { + "key": "", + "plugin": "AMap.Scale,AMap.ToolBar" + }, + "getOption": "const series = [];\n\n// Define your custom color palette with opacity (70%)\nconst colorPalette = [\n 'rgba(118, 150, 137, 0.7)', // '#769689' - Light Green / Forest Light (keep)\n 'rgba(221, 113, 67, 0.7)', // '#dd7143' - Orange (keep)\n 'rgba(154, 96, 180, 0.7)', // '#9A60B4'\n];\n\n// Group the data by 'inc_dist_cohort'\nconst groupedData = context.panel.data.series.reduce((acc, s) => {\n const labelField = s.fields.find((f) => f.name === 'label');\n const cntField = s.fields.find((f) => f.name === 'cnt');\n const timeField = s.fields.find((f) => f.name === 'datetime_label');\n\n // Extract values, handling buffer if present\n const cohort = labelField.values.buffer || labelField.values;\n const cntData = cntField.values.buffer || cntField.values;\n const timeData = timeField.values.buffer || timeField.values;\n\n cohort.forEach((cohortValue, index) => {\n if (!acc[cohortValue]) {\n acc[cohortValue] = [];\n }\n acc[cohortValue].push([timeData[index], cntData[index]]);\n });\n\n return acc;\n}, {});\n\n// Generate a bar series for each cohort with color assignment\nObject.keys(groupedData).forEach((cohortValue, index) => {\n series.push({\n name: `${cohortValue}`, // Label each series as \"Cohort X\"\n type: 'bar',\n stack: 'stack', // Enable stacking\n data: groupedData[cohortValue].map(([x, y]) => [x, parseFloat(y).toFixed(2)]),\n itemStyle: {\n color: colorPalette[index % colorPalette.length], // Assign color cyclically with opacity\n borderColor: 'rgba(26, 26, 26, 1)', // Dark Grey border with full opacity\n borderWidth: 0.3, // Set border width for each bar\n },\n emphasis: { // Highlight on hover\n itemStyle: {\n borderColor: '#000', // Highlight border color (Black)\n borderWidth: 2, // Highlight border width\n },\n },\n });\n});\n\n/**\n * Enable Data Zoom by default\n */\nsetTimeout(() => {\n context.panel.chart.dispatchAction({\n type: 'takeGlobalCursor',\n key: 'dataZoomSelect',\n dataZoomSelectActive: true,\n });\n}, 500);\n\n/**\n * Update Time Range on Zoom\n */\ncontext.panel.chart.on('datazoom', function (params) {\n const startValue = params.batch[0]?.startValue;\n const endValue = params.batch[0]?.endValue;\n locationService.partial({ from: startValue, to: endValue });\n});\n\n// Extract unique datetime_labels for the x-axis\nconst uniqueSlots = Array.from(new Set(\n context.panel.data.series.flatMap(s => {\n const timeField = s.fields.find(f => f.name === 'datetime_label');\n return timeField.values.buffer || timeField.values;\n })\n)).sort((a, b) => a - b); // Sort numerically or lexicographically as needed\n\n// Configure ECharts options with Title, Legend, Tooltip, etc.\nconst option = {\n backgroundColor: '#fffcfa', // Light Cream Background\n\n // Add the Title Component\n title: {\n text: 'Daily Blocks', // Main title text\n subtext: 'State Cohorts', // Optional subtitle text\n left: 'center', // Position the title centrally\n top: '2%', // Position the title slightly below the top edge\n textStyle: {\n fontSize: 18, // Main title font size\n fontWeight: 'bold', // Main title font weight\n color: '#231e10', // Main title color (Black)\n },\n subtextStyle: {\n fontSize: 14, // Subtitle font size\n color: '#3e6957', // Subtitle color (Forest)\n },\n },\n\n tooltip: {\n trigger: 'axis',\n axisPointer: { type: 'shadow' },\n formatter: function (params) {\n // Determine column count dynamically based on number of series\n const columnCount = params.length > 6 ? 3 : 2;\n\n // Start with the axis value (e.g., category name)\n let tooltipText = `\n
\n ${params[0].axisValue}
\n
\n `;\n\n // Iterate through each param to build tooltip entries\n params.forEach(param => {\n let valueAsInt = parseInt(param.value[1], 10);\n tooltipText += `\n
\n ${param.marker} ${param.seriesName}: ${valueAsInt}\n
\n `;\n });\n\n // Close the divs\n tooltipText += `\n
\n
\n `;\n return tooltipText;\n },\n },\n\n legend: {\n data: series.map((s) => s.name),\n top: '18%', // Position the legend below the title\n left: 'center', // Center the legend horizontally\n orient: 'horizontal', // Arrange legend items horizontally\n type: 'scroll', // Enable scrolling when items exceed the container\n pageIconSize: 8, // Reduce the size of scroll buttons to make scrollbar less high\n pageIconInactiveColor: '#ccc', // Inactive scroll button color\n pageIconActiveColor: '#333', // Active scroll button color\n pageTextStyle: {\n color: '#333',\n fontSize: 10, // Smaller font size for page indicators\n },\n width: '60%', // Limit legend width to allow wrapping into multiple lines\n textStyle: {\n fontSize: 12, // Legend text size\n color: '#231e10', // Legend text color (Black)\n },\n itemGap: 15, // Space between legend items\n // itemWidth: 12, // Width of legend symbol (optional)\n // itemHeight: 12, // Height of legend symbol (optional)\n selectedMode: 'multiple', // Allow multiple series to be toggled\n },\n\n xAxis: {\n type: 'category',\n // name: 'Date',\n data: uniqueSlots,\n boundaryGap: true, // Ensure bars are nicely spaced\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize x-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n // axisLabel: {\n // rotate: 45, // Rotate labels for better readability\n // color: '#231e10', // X-axis label color (Black)\n // fontSize: 12, // X-axis label font size\n // },\n },\n\n yAxis: {\n type: 'value',\n name: 'Count',\n axisLabel: {\n formatter: '{value}',\n color: '#231e10', // Y-axis label color (Black)\n fontSize: 12, // Y-axis label font size\n },\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize y-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n splitLine: {\n lineStyle: {\n type: 'dashed',\n color: '#231e10', // Changed from '#cbc3ad' to '#231e10' for better visibility\n width: 1, // Slightly thicker dashed lines\n },\n },\n },\n\n series: series,\n\n dataZoom: [\n {\n type: 'inside', // Enable zooming inside the chart\n start: 60,\n end: 100,\n },\n {\n type: 'slider', // Enable a slider for zooming\n start: 0,\n end: 100,\n height: 10, // Reduce the height of the data zoom slider\n bottom: '10%', // Position it closer to the plot\n handleIcon: 'M10.7,11.9v-1.3H9.3v1.3c-4.9,0.3-8.8,4.4-8.8,9.4c0,5,3.9,9.1,8.8,9.4v1.3h1.4v-1.3c4.9-0.3,8.8-4.4,8.8-9.4C19.5,16.3,15.6,12.2,10.7,11.9z', // Customize handle icon\n handleSize: '80%', // Adjust handle size\n handleStyle: {\n color: '#6f592c', // Handle color (Saddle Brown)\n },\n textStyle: {\n color: '#231e10', // Text color for data zoom\n fontSize: 10, // Text size\n },\n backgroundColor: 'rgba(228, 221, 203, 1)', // '#e4ddcb' - Cream Medium with full opacity\n fillerColor: 'rgba(118, 150, 137, 0.7)', // '#769689' - Light Green with opacity\n borderColor: 'rgba(203, 195, 173, 1)', // '#cbc3ad' - Cream Dark with full opacity\n },\n ],\n\n grid: {\n left: '2%',\n right: '10%',\n top: '25%', // Increased top margin to accommodate legend and title\n bottom: '15%', // Reduced bottom margin to bring the slider closer to the plot\n containLabel: true,\n backgroundColor: '#ffffff', // Plot area's background color (White)\n borderColor: '#cbc3ad', // Border color of the plot area (Cream Dark)\n borderWidth: 1, // Border width of the plot area\n borderRadius: 5, // Rounded corners for the plot area\n shadowColor: 'rgba(0, 0, 0, 0.05)', // Subtle shadow for depth\n shadowBlur: 10,\n shadowOffsetX: 0,\n shadowOffsetY: 2,\n },\n};\n\nreturn option;\n", + "google": { + "callback": "gmapReady", + "key": "" + }, + "map": "none", + "renderer": "canvas", + "themeEditor": { + "config": "{}", + "name": "custom" + }, + "visualEditor": { + "code": "return {\n dataset: context.editor.dataset,\n series: context.editor.series,\n xAxis: {\n type: 'time',\n },\n yAxis: {\n type: 'value',\n min: 'dataMin',\n },\n}\n", + "dataset": [ + { + "name": "f_inclusion_slot", + "source": "A" + }, + { + "name": "cnt", + "source": "A" + } + ], + "series": [] + } + }, + "pluginVersion": "6.4.1", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT \n* \n,CAST(day AS text) AS datetime_label\nFROM dbt.consensus_blocks_production\nORDER BY day ASC, label DESC", + "refId": "A" + } + ], + "type": "volkovlabs-echarts-panel" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 14, + "y": 18 + }, + "id": 21, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n \n

Blocks Production

\n \n

This dashboard provides a comprehensive view of Peer-to-Peer (P2P) network data. Key metrics include the number of active peers, geo location, client diversity and more.

\n
\n", + "mode": "html" + }, + "pluginVersion": "10.1.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 14, + "y": 24 + }, + "id": 7, + "options": { + "baidu": { + "callback": "bmapReady", + "key": "" + }, + "editor": { + "format": "none" + }, + "editorMode": "code", + "gaode": { + "key": "", + "plugin": "AMap.Scale,AMap.ToolBar" + }, + "getOption": "const series = context.panel.data.series.map((s) => {\n const sData = s.fields.find((f) => f.name === 'pct').values.buffer || s.fields.find((f) => f.name === 'pct').values;\n const sTime = s.fields.find((f) => f.type === 'time').values.buffer || s.fields.find((f) => f.type === 'time').values;\n\n return {\n name: \"Participation Rate\", // Updated for clarity\n type: 'line',\n showSymbol: false,\n areaStyle: {\n opacity: 0.1,\n },\n lineStyle: {\n width: 1,\n },\n itemStyle: { // Set the line color\n color: 'rgba(118, 150, 137, 0.7)', // Light Green / Forest Light\n borderColor: 'rgba(26, 26, 26, 1)', // Optional: Dark Grey border with full opacity\n borderWidth: 0.3, // Optional: Set border width for each point\n },\n data: sData.map((d, i) => {\n // If your data is in decimal form, uncomment the next line\n // const value = (d * 100).toFixed(2);\n const value = d.toFixed(2); // Assuming data is already in percentage\n return [sTime[i], value];\n }),\n };\n});\n\n/**\n * Enable Data Zoom by default\n */\nsetTimeout(() => context.panel.chart.dispatchAction({\n type: 'takeGlobalCursor',\n key: 'dataZoomSelect',\n dataZoomSelectActive: true,\n}), 500);\n\n/**\n * Update Time Range on Zoom\n */\ncontext.panel.chart.on('datazoom', function (params) {\n const startValue = params.batch[0]?.startValue;\n const endValue = params.batch[0]?.endValue;\n locationService.partial({ from: startValue, to: endValue });\n});\n\nreturn {\n backgroundColor: '#fffcfa',\n\n // **Title Component**\n title: {\n text: 'Participation Rate', // Main title text\n // subtext: 'Inclusion Distance Cohorts', // Optional subtitle text\n left: 'center', // Position the title centrally\n top: '2%', // Position the title slightly below the top edge\n textStyle: {\n fontSize: 18, // Main title font size\n fontWeight: 'bold', // Main title font weight\n color: '#231e10', // Main title color (Black)\n },\n subtextStyle: {\n fontSize: 14, // Subtitle font size\n color: '#3e6957', // Subtitle color (Forest)\n },\n },\n\n tooltip: {\n trigger: 'axis',\n formatter: function (params) {\n const date = new Date(params[0].data[0]);\n const value = params[0].data[1];\n return `${date.toLocaleDateString()}
Participation Rate: ${value}%`;\n },\n },\n toolbox: {\n feature: {\n dataZoom: {\n yAxisIndex: 'none',\n icon: {\n zoom: 'path://',\n back: 'path://',\n },\n },\n saveAsImage: {},\n }\n },\n xAxis: {\n type: 'time',\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize x-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n },\n\n yAxis: {\n type: 'value',\n min: 0, // Start at 0 for better percentage representation\n max: 100, // Assuming participation rate is out of 100\n axisLabel: {\n formatter: '{value}%', // Append '%' to y-axis labels\n color: '#231e10', // Y-axis label color (Black)\n fontSize: 12, // Y-axis label font size\n },\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize y-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n splitLine: {\n lineStyle: {\n type: 'dashed',\n color: '#231e10', // Changed from '#cbc3ad' to '#231e10' for better visibility\n width: 1, // Slightly thicker dashed lines\n },\n },\n },\n\n dataZoom: [\n {\n type: 'inside', // Enable zooming inside the chart\n start: 0,\n end: 100,\n },\n {\n type: 'slider', // Enable a slider for zooming\n start: 0,\n end: 100,\n height: 10, // Reduce the height of the data zoom slider\n bottom: '10%', // Position it closer to the plot\n handleIcon: 'M10.7,11.9v-1.3H9.3v1.3c-4.9,0.3-8.8,4.4-8.8,9.4c0,5,3.9,9.1,8.8,9.4v1.3h1.4v-1.3c4.9-0.3,8.8-4.4,8.8-9.4C19.5,16.3,15.6,12.2,10.7,11.9z', // Customize handle icon\n handleSize: '80%', // Adjust handle size\n handleStyle: {\n color: '#6f592c', // Handle color (Saddle Brown)\n },\n textStyle: {\n color: '#231e10', // Text color for data zoom\n fontSize: 10, // Text size\n },\n backgroundColor: 'rgba(228, 221, 203, 1)', // '#e4ddcb' - Cream Medium with full opacity\n fillerColor: 'rgba(118, 150, 137, 0.7)', // '#769689' - Light Green with opacity\n borderColor: 'rgba(203, 195, 173, 1)', // '#cbc3ad' - Cream Dark with full opacity\n },\n ],\n\n grid: {\n left: '2%',\n right: '10%',\n top: '25%', // Increased top margin to accommodate legend and title\n bottom: '15%', // Reduced bottom margin to bring the slider closer to the plot\n containLabel: true,\n backgroundColor: '#ffffff', // Plot area's background color (White)\n borderColor: '#cbc3ad', // Border color of the plot area (Cream Dark)\n borderWidth: 1, // Border width of the plot area\n borderRadius: 5, // Rounded corners for the plot area\n shadowColor: 'rgba(0, 0, 0, 0.05)', // Subtle shadow for depth\n shadowBlur: 10,\n shadowOffsetX: 0,\n shadowOffsetY: 2,\n },\n series,\n};\n", + "google": { + "callback": "gmapReady", + "key": "" + }, + "map": "none", + "renderer": "canvas", + "themeEditor": { + "config": "{}", + "name": "custom" + }, + "visualEditor": { + "code": "return {\n dataset: context.editor.dataset,\n series: context.editor.series,\n xAxis: {\n type: 'time',\n },\n yAxis: {\n type: 'value',\n min: 'dataMin',\n },\n}\n", + "dataset": [ + { + "name": "f_inclusion_slot", + "source": "A" + }, + { + "name": "cnt", + "source": "A" + } + ], + "series": [] + } + }, + "pluginVersion": "6.4.1", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT\nday\n,participation_rate * 100 AS pct\n,CAST(day AS text) AS datetime_label\nFROM dbt.consensus_validators_participation_rate\nORDER BY day ASC", + "refId": "A" + } + ], + "type": "volkovlabs-echarts-panel" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 20, + "x": 2, + "y": 30 + }, + "id": 23, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n
", + "mode": "markdown" + }, + "pluginVersion": "10.1.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "gridPos": { + "h": 12, + "w": 14, + "x": 0, + "y": 31 + }, + "id": 4, + "options": { + "baidu": { + "callback": "bmapReady", + "key": "" + }, + "editor": { + "format": "none" + }, + "editorMode": "code", + "gaode": { + "key": "", + "plugin": "AMap.Scale,AMap.ToolBar" + }, + "getOption": "const series = [];\n\n// Define your custom color palette with opacity (70%)\nconst colorPalette = [\n 'rgba(118, 150, 137, 0.7)', // '#769689' - Light Green / Forest Light (keep)\n 'rgba(221, 113, 67, 0.7)', // '#dd7143' - Orange (keep)\n 'rgba(84, 112, 198, 0.7)', // '#5470C6'\n 'rgba(145, 204, 117, 0.7)', // '#91CC75'\n 'rgba(250, 200, 88, 0.7)', // '#FAC858'\n 'rgba(238, 102, 102, 0.7)', // '#EE6666'\n 'rgba(115, 192, 222, 0.7)', // '#73C0DE'\n 'rgba(59, 162, 114, 0.7)', // '#3BA272'\n 'rgba(154, 96, 180, 0.7)', // '#9A60B4'\n 'rgba(234, 124, 204, 0.7)', // '#EA7CCC'\n 'rgba(255, 159, 127, 0.7)', // '#FF9F7F'\n 'rgba(231, 188, 243, 0.7)', // '#E7BCF3'\n 'rgba(212, 130, 101, 0.7)', // '#D48265'\n 'rgba(145, 199, 174, 0.7)', // '#91C7AE'\n 'rgba(116, 159, 131, 0.7)', // '#749F83'\n 'rgba(202, 130, 34, 0.7)', // '#CA8622'\n 'rgba(189, 162, 154, 0.7)', // '#BDA29A'\n];\n\n// Group the data by 'inc_dist_cohort'\nconst groupedData = context.panel.data.series.reduce((acc, s) => {\n const cohortField = s.fields.find((f) => f.name === 'inc_dist_cohort');\n const cntField = s.fields.find((f) => f.name === 'cnt');\n const timeField = s.fields.find((f) => f.name === 'datetime_label');\n\n // Extract values, handling buffer if present\n const cohort = cohortField.values.buffer || cohortField.values;\n const cntData = cntField.values.buffer || cntField.values;\n const timeData = timeField.values.buffer || timeField.values;\n\n cohort.forEach((cohortValue, index) => {\n if (!acc[cohortValue]) {\n acc[cohortValue] = [];\n }\n acc[cohortValue].push([timeData[index], cntData[index]]);\n });\n\n return acc;\n}, {});\n\n// Generate a bar series for each cohort with color assignment\nObject.keys(groupedData).forEach((cohortValue, index) => {\n series.push({\n name: `${cohortValue}`, // Label each series as \"Cohort X\"\n type: 'bar',\n stack: 'stack', // Enable stacking\n data: groupedData[cohortValue].map(([x, y]) => [x, parseFloat(y).toFixed(2)]),\n itemStyle: {\n color: colorPalette[index % colorPalette.length], // Assign color cyclically with opacity\n borderColor: 'rgba(26, 26, 26, 1)', // Dark Grey border with full opacity\n borderWidth: 0.3, // Set border width for each bar\n },\n emphasis: { // Highlight on hover\n itemStyle: {\n borderColor: '#000', // Highlight border color (Black)\n borderWidth: 2, // Highlight border width\n },\n },\n });\n});\n\n/**\n * Enable Data Zoom by default\n */\nsetTimeout(() => {\n context.panel.chart.dispatchAction({\n type: 'takeGlobalCursor',\n key: 'dataZoomSelect',\n dataZoomSelectActive: true,\n });\n}, 500);\n\n/**\n * Update Time Range on Zoom\n */\ncontext.panel.chart.on('datazoom', function (params) {\n const startValue = params.batch[0]?.startValue;\n const endValue = params.batch[0]?.endValue;\n locationService.partial({ from: startValue, to: endValue });\n});\n\n// Extract unique datetime_labels for the x-axis\nconst uniqueSlots = Array.from(new Set(\n context.panel.data.series.flatMap(s => {\n const timeField = s.fields.find(f => f.name === 'datetime_label');\n return timeField.values.buffer || timeField.values;\n })\n)).sort((a, b) => a - b); // Sort numerically or lexicographically as needed\n\n// Configure ECharts options with Title, Legend, Tooltip, etc.\nconst option = {\n backgroundColor: '#fffcfa', // Light Cream Background\n\n // Add the Title Component\n title: {\n text: 'Daily Attestations', // Main title text\n subtext: 'Inclusion Distance Cohorts', // Optional subtitle text\n left: 'center', // Position the title centrally\n top: '2%', // Position the title slightly below the top edge\n textStyle: {\n fontSize: 18, // Main title font size\n fontWeight: 'bold', // Main title font weight\n color: '#231e10', // Main title color (Black)\n },\n subtextStyle: {\n fontSize: 14, // Subtitle font size\n color: '#3e6957', // Subtitle color (Forest)\n },\n },\n\n tooltip: {\n trigger: 'axis',\n axisPointer: { type: 'shadow' },\n formatter: function (params) {\n // Determine column count dynamically based on number of series\n const columnCount = params.length > 6 ? 3 : 2;\n\n // Start with the axis value (e.g., category name)\n let tooltipText = `\n
\n ${params[0].axisValue}
\n
\n `;\n\n // Iterate through each param to build tooltip entries\n params.forEach(param => {\n let valueAsInt = parseInt(param.value[1], 10);\n tooltipText += `\n
\n ${param.marker} ${param.seriesName}: ${valueAsInt}\n
\n `;\n });\n\n // Close the divs\n tooltipText += `\n
\n
\n `;\n return tooltipText;\n },\n },\n\n legend: {\n data: series.map((s) => s.name),\n top: '18%', // Position the legend below the title\n left: 'center', // Center the legend horizontally\n orient: 'horizontal', // Arrange legend items horizontally\n type: 'scroll', // Enable scrolling when items exceed the container\n pageIconSize: 8, // Reduce the size of scroll buttons to make scrollbar less high\n pageIconInactiveColor: '#ccc', // Inactive scroll button color\n pageIconActiveColor: '#333', // Active scroll button color\n pageTextStyle: {\n color: '#333',\n fontSize: 10, // Smaller font size for page indicators\n },\n width: '60%', // Limit legend width to allow wrapping into multiple lines\n textStyle: {\n fontSize: 12, // Legend text size\n color: '#231e10', // Legend text color (Black)\n },\n itemGap: 15, // Space between legend items\n // itemWidth: 12, // Width of legend symbol (optional)\n // itemHeight: 12, // Height of legend symbol (optional)\n selectedMode: 'multiple', // Allow multiple series to be toggled\n },\n\n xAxis: {\n type: 'category',\n // name: 'Date',\n data: uniqueSlots,\n boundaryGap: true, // Ensure bars are nicely spaced\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize x-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n // axisLabel: {\n // rotate: 45, // Rotate labels for better readability\n // color: '#231e10', // X-axis label color (Black)\n // fontSize: 12, // X-axis label font size\n // },\n },\n\n yAxis: {\n type: 'value',\n name: 'Count',\n axisLabel: {\n formatter: '{value}',\n color: '#231e10', // Y-axis label color (Black)\n fontSize: 12, // Y-axis label font size\n },\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize y-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n splitLine: {\n lineStyle: {\n type: 'dashed',\n color: '#231e10', // Changed from '#cbc3ad' to '#231e10' for better visibility\n width: 1, // Slightly thicker dashed lines\n },\n },\n },\n\n series: series,\n\n dataZoom: [\n {\n type: 'inside', // Enable zooming inside the chart\n start: 60,\n end: 100,\n },\n {\n type: 'slider', // Enable a slider for zooming\n start: 0,\n end: 100,\n height: 10, // Reduce the height of the data zoom slider\n bottom: '10%', // Position it closer to the plot\n handleIcon: 'M10.7,11.9v-1.3H9.3v1.3c-4.9,0.3-8.8,4.4-8.8,9.4c0,5,3.9,9.1,8.8,9.4v1.3h1.4v-1.3c4.9-0.3,8.8-4.4,8.8-9.4C19.5,16.3,15.6,12.2,10.7,11.9z', // Customize handle icon\n handleSize: '80%', // Adjust handle size\n handleStyle: {\n color: '#6f592c', // Handle color (Saddle Brown)\n },\n textStyle: {\n color: '#231e10', // Text color for data zoom\n fontSize: 10, // Text size\n },\n backgroundColor: 'rgba(228, 221, 203, 1)', // '#e4ddcb' - Cream Medium with full opacity\n fillerColor: 'rgba(118, 150, 137, 0.7)', // '#769689' - Light Green with opacity\n borderColor: 'rgba(203, 195, 173, 1)', // '#cbc3ad' - Cream Dark with full opacity\n },\n ],\n\n grid: {\n left: '2%',\n right: '10%',\n top: '25%', // Increased top margin to accommodate legend and title\n bottom: '15%', // Reduced bottom margin to bring the slider closer to the plot\n containLabel: true,\n backgroundColor: '#ffffff', // Plot area's background color (White)\n borderColor: '#cbc3ad', // Border color of the plot area (Cream Dark)\n borderWidth: 1, // Border width of the plot area\n borderRadius: 5, // Rounded corners for the plot area\n shadowColor: 'rgba(0, 0, 0, 0.05)', // Subtle shadow for depth\n shadowBlur: 10,\n shadowOffsetX: 0,\n shadowOffsetY: 2,\n },\n};\n\nreturn option;\n", + "google": { + "callback": "gmapReady", + "key": "" + }, + "map": "none", + "renderer": "canvas", + "themeEditor": { + "config": "{}", + "name": "custom" + }, + "visualEditor": { + "code": "return {\n dataset: context.editor.dataset,\n series: context.editor.series,\n xAxis: {\n type: 'time',\n },\n yAxis: {\n type: 'value',\n min: 'dataMin',\n },\n}\n", + "dataset": [ + { + "name": "f_inclusion_slot", + "source": "A" + }, + { + "name": "cnt", + "source": "A" + } + ], + "series": [] + } + }, + "pluginVersion": "6.4.1", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT \n* \n,CAST(day AS text) AS datetime_label\nFROM dbt.consensus_atts_inclusion_distance_d\nORDER BY day ASC", + "refId": "A" + } + ], + "type": "volkovlabs-echarts-panel" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 14, + "y": 31 + }, + "id": 24, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n \n

Network Attestations

\n \n

This dashboard provides a comprehensive view of Peer-to-Peer (P2P) network data. Key metrics include the number of active peers, geo location, client diversity and more.

\n
\n", + "mode": "html" + }, + "pluginVersion": "10.1.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 20, + "x": 2, + "y": 43 + }, + "id": 25, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n
", + "mode": "markdown" + }, + "pluginVersion": "10.1.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "gridPos": { + "h": 6, + "w": 14, + "x": 0, + "y": 44 + }, + "id": 10, + "options": { + "baidu": { + "callback": "bmapReady", + "key": "" + }, + "editor": { + "format": "none" + }, + "editorMode": "code", + "gaode": { + "key": "", + "plugin": "AMap.Scale,AMap.ToolBar" + }, + "getOption": "const series = context.panel.data.series.map((s) => {\n const sData = s.fields.find((f) => f.type === 'number').values.buffer || s.fields.find((f) => f.type === 'number').values;\n const sTime = s.fields.find((f) => f.type === 'time').values.buffer || s.fields.find((f) => f.type === 'time').values;\n\n return {\n name: \"nodes\",\n type: 'bar',\n showSymbol: false,\n areaStyle: {\n opacity: 0.1,\n },\n lineStyle: {\n width: 1,\n },\n itemStyle: { // Set the bar color\n color: 'rgba(118, 150, 137, 0.7)', // Light Green / Forest Light\n borderColor: 'rgba(26, 26, 26, 1)', // Optional: Dark Grey border with full opacity\n borderWidth: 0.3, // Optional: Set border width for each bar\n },\n data: sData.map((d, i) => [sTime[i], d.toFixed(2)]),\n };\n});\n\n/**\n * Enable Data Zoom by default\n */\nsetTimeout(() => context.panel.chart.dispatchAction({\n type: 'takeGlobalCursor',\n key: 'dataZoomSelect',\n dataZoomSelectActive: true,\n}), 500);\n\n/**\n * Update Time Range on Zoom\n */\ncontext.panel.chart.on('datazoom', function (params) {\n const startValue = params.batch[0]?.startValue;\n const endValue = params.batch[0]?.endValue;\n locationService.partial({ from: startValue, to: endValue });\n});\n\nreturn {\n backgroundColor: '#fffcfa',\n\n // **Title Component**\n title: {\n text: 'Validators Entry Waiting Time', // Main title text\n // subtext: 'Inclusion Distance Cohorts', // Optional subtitle text\n left: 'center', // Position the title centrally\n top: '2%', // Position the title slightly below the top edge\n textStyle: {\n fontSize: 18, // Main title font size\n fontWeight: 'bold', // Main title font weight\n color: '#231e10', // Main title color (Black)\n },\n subtextStyle: {\n fontSize: 14, // Subtitle font size\n color: '#3e6957', // Subtitle color (Forest)\n },\n },\n\n tooltip: {\n trigger: 'axis',\n },\n toolbox: {\n feature: {\n dataZoom: {\n yAxisIndex: 'none',\n icon: {\n zoom: 'path://',\n back: 'path://',\n },\n },\n saveAsImage: {},\n }\n },\n xAxis: {\n type: 'time',\n name: 'Day',\n nameLocation: 'middle',\n nameGap: 20,\n boundaryGap: true,\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize x-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n },\n\n yAxis: {\n type: 'value',\n min: 'dataMin',\n name: 'Hours',\n axisLabel: {\n formatter: '{value}',\n color: '#231e10', // Y-axis label color (Black)\n fontSize: 12, // Y-axis label font size\n },\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize y-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n splitLine: {\n lineStyle: {\n type: 'dashed',\n color: '#231e10', // Changed from '#cbc3ad' to '#231e10' for better visibility\n width: 1, // Slightly thicker dashed lines\n },\n },\n },\n\n dataZoom: [\n {\n type: 'inside', // Enable zooming inside the chart\n start: 80,\n end: 100,\n },\n {\n type: 'slider', // Enable a slider for zooming\n start: 0,\n end: 100,\n height: 10, // Reduce the height of the data zoom slider\n bottom: '10%', // Position it closer to the plot\n handleIcon: 'M10.7,11.9v-1.3H9.3v1.3c-4.9,0.3-8.8,4.4-8.8,9.4c0,5,3.9,9.1,8.8,9.4v1.3h1.4v-1.3c4.9-0.3,8.8-4.4,8.8-9.4C19.5,16.3,15.6,12.2,10.7,11.9z', // Customize handle icon\n handleSize: '80%', // Adjust handle size\n handleStyle: {\n color: '#6f592c', // Handle color (Saddle Brown)\n },\n textStyle: {\n color: '#231e10', // Text color for data zoom\n fontSize: 10, // Text size\n },\n backgroundColor: 'rgba(228, 221, 203, 1)', // '#e4ddcb' - Cream Medium with full opacity\n fillerColor: 'rgba(118, 150, 137, 0.7)', // '#769689' - Light Green with opacity\n borderColor: 'rgba(203, 195, 173, 1)', // '#cbc3ad' - Cream Dark with full opacity\n },\n ],\n\n grid: {\n left: '2%',\n right: '10%',\n top: '25%', // Increased top margin to accommodate legend and title\n bottom: '15%', // Reduced bottom margin to bring the slider closer to the plot\n containLabel: true,\n backgroundColor: '#ffffff', // Plot area's background color (White)\n borderColor: '#cbc3ad', // Border color of the plot area (Cream Dark)\n borderWidth: 1, // Border width of the plot area\n borderRadius: 5, // Rounded corners for the plot area\n shadowColor: 'rgba(0, 0, 0, 0.05)', // Subtle shadow for depth\n shadowBlur: 10,\n shadowOffsetX: 0,\n shadowOffsetY: 2,\n },\n series,\n};\n", + "google": { + "callback": "gmapReady", + "key": "" + }, + "map": "none", + "renderer": "canvas", + "themeEditor": { + "config": "{}", + "name": "custom" + }, + "visualEditor": { + "code": "return {\n dataset: context.editor.dataset,\n series: context.editor.series,\n xAxis: {\n type: 'time',\n },\n yAxis: {\n type: 'value',\n min: 'dataMin',\n },\n}\n", + "dataset": [ + { + "name": "f_inclusion_slot", + "source": "A" + }, + { + "name": "cnt", + "source": "A" + } + ], + "series": [] + } + }, + "pluginVersion": "6.4.1", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT \n* \n,CAST(day AS text) AS datetime_label\nFROM dbt.consensus_validators_waiting_times\nWHERE label = 'Median Entry'\nORDER BY day ASC", + "refId": "A" + } + ], + "type": "volkovlabs-echarts-panel" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 14, + "y": 44 + }, + "id": 26, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n \n

Waiting Times

\n \n

This dashboard provides a comprehensive view of Peer-to-Peer (P2P) network data. Key metrics include the number of active peers, geo location, client diversity and more.

\n
\n", + "mode": "html" + }, + "pluginVersion": "10.1.0", + "transparent": true, + "type": "text" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "gridPos": { + "h": 6, + "w": 14, + "x": 0, + "y": 50 + }, + "id": 12, + "options": { + "baidu": { + "callback": "bmapReady", + "key": "" + }, + "editor": { + "format": "none" + }, + "editorMode": "code", + "gaode": { + "key": "", + "plugin": "AMap.Scale,AMap.ToolBar" + }, + "getOption": "const series = context.panel.data.series.map((s) => {\n const sData = s.fields.find((f) => f.type === 'number').values.buffer || s.fields.find((f) => f.type === 'number').values;\n const sTime = s.fields.find((f) => f.type === 'time').values.buffer || s.fields.find((f) => f.type === 'time').values;\n\n return {\n name: \"nodes\",\n type: 'bar',\n showSymbol: false,\n areaStyle: {\n opacity: 0.1,\n },\n lineStyle: {\n width: 1,\n },\n itemStyle: { // Set the bar color\n color: 'rgba(118, 150, 137, 0.7)', // Light Green / Forest Light\n borderColor: 'rgba(26, 26, 26, 1)', // Optional: Dark Grey border with full opacity\n borderWidth: 0.3, // Optional: Set border width for each bar\n },\n data: sData.map((d, i) => [sTime[i], d.toFixed(2)]),\n };\n});\n\n/**\n * Enable Data Zoom by default\n */\nsetTimeout(() => context.panel.chart.dispatchAction({\n type: 'takeGlobalCursor',\n key: 'dataZoomSelect',\n dataZoomSelectActive: true,\n}), 500);\n\n/**\n * Update Time Range on Zoom\n */\ncontext.panel.chart.on('datazoom', function (params) {\n const startValue = params.batch[0]?.startValue;\n const endValue = params.batch[0]?.endValue;\n locationService.partial({ from: startValue, to: endValue });\n});\n\nreturn {\n backgroundColor: '#fffcfa',\n\n // **Title Component**\n title: {\n text: 'Validators Exit Waiting Time', // Main title text\n // subtext: 'Inclusion Distance Cohorts', // Optional subtitle text\n left: 'center', // Position the title centrally\n top: '2%', // Position the title slightly below the top edge\n textStyle: {\n fontSize: 18, // Main title font size\n fontWeight: 'bold', // Main title font weight\n color: '#231e10', // Main title color (Black)\n },\n subtextStyle: {\n fontSize: 14, // Subtitle font size\n color: '#3e6957', // Subtitle color (Forest)\n },\n },\n\n tooltip: {\n trigger: 'axis',\n },\n toolbox: {\n feature: {\n dataZoom: {\n yAxisIndex: 'none',\n icon: {\n zoom: 'path://',\n back: 'path://',\n },\n },\n saveAsImage: {},\n }\n },\n xAxis: {\n type: 'time',\n name: 'Day',\n nameLocation: 'middle',\n nameGap: 20,\n boundaryGap: true,\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize x-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n },\n\n yAxis: {\n type: 'value',\n min: 'dataMin',\n name: 'Hours',\n axisLabel: {\n formatter: '{value}',\n color: '#231e10', // Y-axis label color (Black)\n fontSize: 12, // Y-axis label font size\n },\n axisLine: {\n lineStyle: {\n color: '#231e10', // Customize y-axis line color (Black)\n width: 2, // Increased line width for better visibility\n },\n },\n splitLine: {\n lineStyle: {\n type: 'dashed',\n color: '#231e10', // Changed from '#cbc3ad' to '#231e10' for better visibility\n width: 1, // Slightly thicker dashed lines\n },\n },\n },\n\n dataZoom: [\n {\n type: 'inside', // Enable zooming inside the chart\n start: 80,\n end: 100,\n },\n {\n type: 'slider', // Enable a slider for zooming\n start: 0,\n end: 100,\n height: 10, // Reduce the height of the data zoom slider\n bottom: '10%', // Position it closer to the plot\n handleIcon: 'M10.7,11.9v-1.3H9.3v1.3c-4.9,0.3-8.8,4.4-8.8,9.4c0,5,3.9,9.1,8.8,9.4v1.3h1.4v-1.3c4.9-0.3,8.8-4.4,8.8-9.4C19.5,16.3,15.6,12.2,10.7,11.9z', // Customize handle icon\n handleSize: '80%', // Adjust handle size\n handleStyle: {\n color: '#6f592c', // Handle color (Saddle Brown)\n },\n textStyle: {\n color: '#231e10', // Text color for data zoom\n fontSize: 10, // Text size\n },\n backgroundColor: 'rgba(228, 221, 203, 1)', // '#e4ddcb' - Cream Medium with full opacity\n fillerColor: 'rgba(118, 150, 137, 0.7)', // '#769689' - Light Green with opacity\n borderColor: 'rgba(203, 195, 173, 1)', // '#cbc3ad' - Cream Dark with full opacity\n },\n ],\n\n grid: {\n left: '2%',\n right: '10%',\n top: '25%', // Increased top margin to accommodate legend and title\n bottom: '15%', // Reduced bottom margin to bring the slider closer to the plot\n containLabel: true,\n backgroundColor: '#ffffff', // Plot area's background color (White)\n borderColor: '#cbc3ad', // Border color of the plot area (Cream Dark)\n borderWidth: 1, // Border width of the plot area\n borderRadius: 5, // Rounded corners for the plot area\n shadowColor: 'rgba(0, 0, 0, 0.05)', // Subtle shadow for depth\n shadowBlur: 10,\n shadowOffsetX: 0,\n shadowOffsetY: 2,\n },\n series,\n};\n", + "google": { + "callback": "gmapReady", + "key": "" + }, + "map": "none", + "renderer": "canvas", + "themeEditor": { + "config": "{}", + "name": "custom" + }, + "visualEditor": { + "code": "return {\n dataset: context.editor.dataset,\n series: context.editor.series,\n xAxis: {\n type: 'time',\n },\n yAxis: {\n type: 'value',\n min: 'dataMin',\n },\n}\n", + "dataset": [ + { + "name": "f_inclusion_slot", + "source": "A" + }, + { + "name": "cnt", + "source": "A" + } + ], + "series": [] + } + }, + "pluginVersion": "6.4.1", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT \n* \n,CAST(day AS text) AS datetime_label\nFROM dbt.consensus_validators_waiting_times\nWHERE label = 'Median Exit'\nORDER BY day ASC", + "refId": "A" + } + ], + "type": "volkovlabs-echarts-panel" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#3e6957", + "value": null + }, + { + "color": "#dd7143b3", + "value": 24 + }, + { + "color": "#eb5e38", + "value": 48 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 14, + "y": 50 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT \navg(value) AS value\nFROM dbt.consensus_validators_waiting_times\nWHERE label = 'Median Entry'\nAND day >= (SELECT MAX(day) FROM dbt.consensus_validators_waiting_times) - INTERVAL 7 DAYS\n", + "refId": "A" + } + ], + "title": "Last 7 Days Median Entry Time", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#3e6957", + "value": null + }, + { + "color": "#dd7143b3", + "value": 6 + }, + { + "color": "#eb5e38", + "value": 16 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 14, + "y": 53 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.0", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PD136F4A5A3B015AF" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.0", + "queryType": "table", + "rawSql": "SELECT \navg(value) AS value\nFROM dbt.consensus_validators_waiting_times\nWHERE label = 'Median Exit'\nAND day >= ((SELECT MAX(day) FROM dbt.consensus_validators_waiting_times) - INTERVAL 14 DAYS)", + "refId": "A" + } + ], + "title": "Last 14 Days Median Exit Time", + "type": "stat" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Consensus Overview", + "uid": "b0e1d65b-c5d9-4f58-9e2b-9c29779c2c53", + "version": 28, + "weekStart": "" + } +} diff --git a/grafana/grafana_dashboards_backup/c049b211-0058-4301-b9e2-aa54973022e1.json b/grafana/grafana_dashboards_backup/c049b211-0058-4301-b9e2-aa54973022e1.json new file mode 100644 index 0000000..7155d05 --- /dev/null +++ b/grafana/grafana_dashboards_backup/c049b211-0058-4301-b9e2-aa54973022e1.json @@ -0,0 +1,48 @@ +{ + "meta": { + "type": "db", + "canSave": true, + "canEdit": true, + "canAdmin": true, + "canStar": true, + "canDelete": true, + "slug": "cerebro-dashboards", + "url": "/dashboards/f/c049b211-0058-4301-b9e2-aa54973022e1/cerebro-dashboards", + "expires": "0001-01-01T00:00:00Z", + "created": "2024-10-04T10:41:20Z", + "updated": "2024-10-04T10:41:20Z", + "updatedBy": "gnosisid_hugo.serodio@gnosis.io", + "createdBy": "gnosisid_hugo.serodio@gnosis.io", + "version": 1, + "hasAcl": false, + "isFolder": true, + "folderId": 0, + "folderUid": "", + "folderTitle": "General", + "folderUrl": "", + "provisioned": false, + "provisionedExternalId": "", + "annotationsPermissions": { + "dashboard": { + "canAdd": true, + "canEdit": true, + "canDelete": true + }, + "organization": { + "canAdd": true, + "canEdit": true, + "canDelete": true + } + }, + "publicDashboardAccessToken": "", + "publicDashboardUid": "", + "publicDashboardEnabled": false + }, + "dashboard": { + "id": 2, + "schemaVersion": 17, + "title": "Cerebro Dashboards", + "uid": "c049b211-0058-4301-b9e2-aa54973022e1", + "version": 1 + } +} diff --git a/grafana/grafana_dashboards_backup/h73c933j-09fc-095c-9185-26e92003e016.json b/grafana/grafana_dashboards_backup/h73c933j-09fc-095c-9185-26e92003e016.json new file mode 100644 index 0000000..d4b902b --- /dev/null +++ b/grafana/grafana_dashboards_backup/h73c933j-09fc-095c-9185-26e92003e016.json @@ -0,0 +1,412 @@ +{ + "meta": { + "type": "db", + "canSave": true, + "canEdit": true, + "canAdmin": true, + "canStar": true, + "canDelete": true, + "slug": "logs-search", + "url": "/d/h73c933j-09fc-095c-9185-26e92003e016/logs-search", + "expires": "0001-01-01T00:00:00Z", + "created": "2024-10-04T08:40:36Z", + "updated": "2024-10-04T08:40:36Z", + "updatedBy": "Anonymous", + "createdBy": "Anonymous", + "version": 1, + "hasAcl": false, + "isFolder": false, + "folderId": 0, + "folderUid": "", + "folderTitle": "General", + "folderUrl": "", + "provisioned": false, + "provisionedExternalId": "logs-search.json", + "annotationsPermissions": { + "dashboard": { + "canAdd": true, + "canEdit": true, + "canDelete": true + }, + "organization": { + "canAdd": true, + "canEdit": true, + "canDelete": true + } + }, + "publicDashboardAccessToken": "", + "publicDashboardUid": "", + "publicDashboardEnabled": false + }, + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Loki logs search", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 12019, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "loki", + "uid": "loki-gnosis" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.0", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki-gnosis" + }, + "expr": "sum(count_over_time({job=~\"$job\"} |~ \"$user_id\" |~ \"$level\" |~ \"$search\"[$__interval]))", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Logs Interval Panel", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:168", + "format": "short", + "logBase": 1, + "show": false + }, + { + "$$hashKey": "object:169", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "loki", + "uid": "loki-gnosis" + }, + "gridPos": { + "h": 25, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 2, + "maxDataPoints": "", + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki-gnosis" + }, + "editorMode": "code", + "expr": "{job=~\"$job\"} |~ \"$user_id\" |~ \"$level\" |~ \"$search\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Logs Panel", + "type": "logs" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "LOKI", + "LOGS" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "aws-gnosis", + "value": "aws-gnosis" + }, + "datasource": { + "type": "cloudwatch", + "uid": "aws-gnosis" + }, + "description": null, + "error": null, + "hide": 2, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "cloudwatch", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "default", + "value": "default" + }, + "datasource": { + "type": "cloudwatch", + "uid": "aws-gnosis" + }, + "definition": "", + "description": null, + "error": null, + "hide": 2, + "includeAll": false, + "label": "Region", + "multi": false, + "name": "region", + "options": [], + "query": "regions()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "gnosis-prod", + "value": "gnosis-prod" + }, + "datasource": { + "type": "cloudwatch", + "uid": "aws-gnosis" + }, + "definition": "", + "description": null, + "error": null, + "hide": 2, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "dimension_values($region,AWS/ECS,CPUUtilization,ClusterName)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "grafana", + "value": "grafana" + }, + "datasource": { + "type": "cloudwatch", + "uid": "aws-gnosis" + }, + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": "dimension_values($region,AWS/ECS,CPUUtilization,ServiceName,{\"ClusterName\":\"gnosis-prod\"})", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "label": "User ID", + "name": "user_id", + "options": [], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "includeAll": false, + "label": "Log Level", + "multi": true, + "name": "level", + "options": [ + { + "selected": true, + "text": "ANY", + "value": "" + }, + { + "text": "TRACE", + "value": "trace" + }, + { + "text": "DEBUG", + "value": "debug" + }, + { + "text": "INFO", + "value": "info" + }, + { + "text": "WARNING", + "value": "warn" + }, + { + "text": "ERROR", + "value": "error" + }, + { + "text": "CRITICAL", + "value": "critical" + } + ], + "query": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "label": "Search", + "name": "search", + "options": [], + "query": "", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Logs Search", + "uid": "h73c933j-09fc-095c-9185-26e92003e016", + "version": 1, + "weekStart": "" + } +} diff --git a/models/metrics/validators/consensus_validators_waiting_times.sql b/models/metrics/validators/consensus_validators_waiting_times.sql new file mode 100644 index 0000000..bf2b52b --- /dev/null +++ b/models/metrics/validators/consensus_validators_waiting_times.sql @@ -0,0 +1,30 @@ +{{ + config( + materialized='table' + ) +}} + +WITH +final AS ( + SELECT + toDate(activation_eligibility_time) AS day, + toInt64(activation_time) - toInt64(activation_eligibility_time) AS entry_delay, + toInt64(withdrawable_time) - toInt64(exit_time) AS exit_delay + FROM {{ ref('consensus_validators_status') }} +) + +SELECT day, COALESCE(toFloat64(min(entry_delay)),0)/3600 AS value, 'Min Entry' AS label FROM final GROUP BY 1 +UNION ALL +SELECT day, COALESCE(toFloat64(max(entry_delay)),0)/3600 AS value, 'Max Entry' AS label FROM final GROUP BY 1 +UNION ALL +SELECT day, COALESCE(toFloat64(median(entry_delay)),0)/3600 AS value, 'Median Entry' AS label FROM final GROUP BY 1 +UNION ALL +SELECT day, COALESCE(toFloat64(avg(entry_delay)),0)/3600 AS value, 'Mean Entry' AS label FROM final GROUP BY 1 +UNION ALL +SELECT day, COALESCE(toFloat64(min(exit_delay)),0)/3600 AS value, 'Min Exit' AS label FROM final GROUP BY 1 +UNION ALL +SELECT day, COALESCE(toFloat64(max(exit_delay)),0)/3600 AS value, 'Max Exit' AS label FROM final GROUP BY 1 +UNION ALL +SELECT day, COALESCE(toFloat64(median(exit_delay)),0)/3600 AS value, 'Median Exit' AS label FROM final GROUP BY 1 +UNION ALL +SELECT day, COALESCE(toFloat64(avg(exit_delay)),0)/3600 AS value, 'Mean Exit' AS label FROM final GROUP BY 1 \ No newline at end of file diff --git a/models/sources/p2p/p2p_sources.yml b/models/sources/p2p/p2p_sources.yml index e2fcbe5..30bfaf2 100644 --- a/models/sources/p2p/p2p_sources.yml +++ b/models/sources/p2p/p2p_sources.yml @@ -15,3 +15,12 @@ sources: - name: t_proposer_duties - name: t_blocks - name: t_attestations + + - name: chaind + database: chaind + schema: public + tables: + - name: t_proposer_duties + - name: t_blocks + - name: t_attestations + diff --git a/models/transformations/validatros/consensus_validators_status.sql b/models/transformations/validators/consensus_validators_status.sql similarity index 100% rename from models/transformations/validatros/consensus_validators_status.sql rename to models/transformations/validators/consensus_validators_status.sql diff --git a/models/transformations/validators/test_chaind.sql b/models/transformations/validators/test_chaind.sql new file mode 100644 index 0000000..ff5c10e --- /dev/null +++ b/models/transformations/validators/test_chaind.sql @@ -0,0 +1,16 @@ +{{ + config( + materialized='table' + ) +}} + +WITH + +validators AS ( + SELECT + COUNT(*) + FROM + {{ get_postgres('chaind', 't_attestations') }} +) + +SELECT * FROM validators \ No newline at end of file diff --git a/profiles.yml b/profiles.yml index aef07b5..963b5e5 100644 --- a/profiles.yml +++ b/profiles.yml @@ -22,4 +22,14 @@ gnosis_dbt: user: "{{ env_var('POSTGRES_USER') }}" password: "{{ env_var('POSTGRES_PASSWORD') }}" port: "{{ env_var('POSTGRES_PORT', '5432') | int }}" + threads: 80 + + chaind: + type: postgres + dbname: "chaind" + schema: "public" + host: "{{ env_var('POSTGRES_HOST', 'postgres') }}" + user: "{{ env_var('POSTGRES_USER') }}" + password: "{{ env_var('POSTGRES_PASSWORD') }}" + port: "{{ env_var('POSTGRES_PORT', '5432') | int }}" threads: 80 \ No newline at end of file From 395b3ede60b2841937e0c2dc70ea97d35b391264 Mon Sep 17 00:00:00 2001 From: hdser Date: Fri, 1 Nov 2024 16:48:04 +0100 Subject: [PATCH 3/4] models and structure updated --- .../consensus/compute_timestamp_at_slot.sql | 19 ++ macros/consensus/decode_graffiti.sql | 11 + macros/consensus/get_chain_spec.sql | 3 + .../seconds_until_end_of_day.sql | 0 macros/db/compute_timestamp_at_slot.sql | 19 -- macros/db/get_chain_spec.sql | 3 - .../consensus_atts_inclusion_distance_d.sql | 28 +++ .../consensus/attestations/metrics/schema.yml | 38 +++ .../consensus_blocks_graffiti_top10_d.sql | 62 +++++ .../consensus_blocks_graffiti_validators.sql | 119 ++++++++++ .../metrics}/consensus_blocks_production.sql | 12 +- models/consensus/blocks/metrics/schema.yml | 85 +++++++ .../consensus_blocks_graffiti.sql | 28 +++ .../blocks/transformations/schema.yml | 43 ++++ models/consensus/consensus_sources.yml | 30 +++ .../consensus_validators_activations.sql | 8 +- .../consensus_validators_eff_balance_dist.sql | 44 ++++ .../consensus_validators_entry_queue.sql | 86 +++++++ ...onsensus_validators_participation_rate.sql | 31 +++ .../consensus_validators_queue_cnt_d.sql | 56 +++++ .../consensus_validators_waiting_times.sql | 56 +++++ .../consensus/validators/metrics/schema.yml | 223 ++++++++++++++++++ .../consensus_validators_queue.sql | 45 ++++ .../validators/transformations/schema.yml | 98 ++++++++ .../gnosis_carbon_emissions.sql | 51 ---- .../gnosis_power_consumption.sql | 113 --------- .../consensus_atts_inclusion_distance_d.sql | 24 -- .../metrics/p2p/gnosis_p2p_nodes_forks.sqlxx | 11 - models/metrics/p2p/gnosis_p2p_nodes_geo.sqlxx | 11 - .../metrics/p2p/gnosis_p2p_nodes_host.sqlxx | 11 - .../metrics/p2p/p2p_valtrack_nodes_cnt_1d.sql | 1 - .../metrics/p2p/p2p_valtrack_nodes_cnt_1h.sql | 1 - .../p2p/p2p_valtrack_nodes_cnt_country_1d.sql | 1 - .../p2p/p2p_valtrack_nodes_geo_last_day.sql | 26 -- .../consensus_validators_participation.sql | 34 --- ...onsensus_validators_participation_rate.sql | 40 ---- .../consensus_validators_waiting_times.sql | 30 --- models/p2p/p2p_sources.yml | 10 + models/sources/p2p/p2p_sources.yml | 26 -- .../consensus_atts_inclusion_distance.sql | 65 ----- .../consensus_atts_inclusion_distance_old.sql | 66 ------ .../p2p/gnosis_p2p_network.yml | 74 ------ .../p2p/gnosis_p2p_nodes_status.sql | 53 ----- models/transformations/p2p/test.sql | 14 -- models/transformations/p2p/test2.sql | 7 - .../power_consumption/consensus_power.sql | 14 -- .../power_consumption/execution_power.sql | 15 -- .../power_consumption/gnosis_nodes_geo.sql | 11 - .../power_consumption/hardware_config.sql | 28 --- .../power_consumption/idle_electric_power.sql | 12 - .../power_consumption/node_distribution.sql | 12 - .../consensus_validators_status.sql | 25 -- .../validators/test_chaind.sql | 16 -- profiles.yml | 10 - scripts/ymls/requirements.txt | 2 + scripts/ymls/ymls_generator.py | 162 +++++++++++++ 56 files changed, 1289 insertions(+), 834 deletions(-) create mode 100644 macros/consensus/compute_timestamp_at_slot.sql create mode 100644 macros/consensus/decode_graffiti.sql create mode 100644 macros/consensus/get_chain_spec.sql rename macros/{db => consensus}/seconds_until_end_of_day.sql (100%) delete mode 100644 macros/db/compute_timestamp_at_slot.sql delete mode 100644 macros/db/get_chain_spec.sql create mode 100644 models/consensus/attestations/metrics/consensus_atts_inclusion_distance_d.sql create mode 100644 models/consensus/attestations/metrics/schema.yml create mode 100644 models/consensus/blocks/metrics/consensus_blocks_graffiti_top10_d.sql create mode 100644 models/consensus/blocks/metrics/consensus_blocks_graffiti_validators.sql rename models/{transformations/attestations => consensus/blocks/metrics}/consensus_blocks_production.sql (75%) create mode 100644 models/consensus/blocks/metrics/schema.yml create mode 100644 models/consensus/blocks/transformations/consensus_blocks_graffiti.sql create mode 100644 models/consensus/blocks/transformations/schema.yml create mode 100644 models/consensus/consensus_sources.yml rename models/{metrics/validators => consensus/validators/metrics}/consensus_validators_activations.sql (84%) create mode 100644 models/consensus/validators/metrics/consensus_validators_eff_balance_dist.sql create mode 100644 models/consensus/validators/metrics/consensus_validators_entry_queue.sql create mode 100644 models/consensus/validators/metrics/consensus_validators_participation_rate.sql create mode 100644 models/consensus/validators/metrics/consensus_validators_queue_cnt_d.sql create mode 100644 models/consensus/validators/metrics/consensus_validators_waiting_times.sql create mode 100644 models/consensus/validators/metrics/schema.yml create mode 100644 models/consensus/validators/transformations/consensus_validators_queue.sql create mode 100644 models/consensus/validators/transformations/schema.yml delete mode 100644 models/metrics/_carbon_ratings/gnosis_carbon_emissions.sql delete mode 100644 models/metrics/_carbon_ratings/gnosis_power_consumption.sql delete mode 100644 models/metrics/attestations/consensus_atts_inclusion_distance_d.sql delete mode 100644 models/metrics/p2p/gnosis_p2p_nodes_forks.sqlxx delete mode 100644 models/metrics/p2p/gnosis_p2p_nodes_geo.sqlxx delete mode 100644 models/metrics/p2p/gnosis_p2p_nodes_host.sqlxx delete mode 100644 models/metrics/p2p/p2p_valtrack_nodes_cnt_1d.sql delete mode 100644 models/metrics/p2p/p2p_valtrack_nodes_cnt_1h.sql delete mode 100644 models/metrics/p2p/p2p_valtrack_nodes_cnt_country_1d.sql delete mode 100644 models/metrics/p2p/p2p_valtrack_nodes_geo_last_day.sql delete mode 100644 models/metrics/validators/consensus_validators_participation.sql delete mode 100644 models/metrics/validators/consensus_validators_participation_rate.sql delete mode 100644 models/metrics/validators/consensus_validators_waiting_times.sql create mode 100644 models/p2p/p2p_sources.yml delete mode 100644 models/sources/p2p/p2p_sources.yml delete mode 100644 models/transformations/attestations/consensus_atts_inclusion_distance.sql delete mode 100644 models/transformations/attestations/consensus_atts_inclusion_distance_old.sql delete mode 100644 models/transformations/p2p/gnosis_p2p_network.yml delete mode 100644 models/transformations/p2p/gnosis_p2p_nodes_status.sql delete mode 100644 models/transformations/p2p/test.sql delete mode 100644 models/transformations/p2p/test2.sql delete mode 100644 models/transformations/power_consumption/consensus_power.sql delete mode 100644 models/transformations/power_consumption/execution_power.sql delete mode 100644 models/transformations/power_consumption/gnosis_nodes_geo.sql delete mode 100644 models/transformations/power_consumption/hardware_config.sql delete mode 100644 models/transformations/power_consumption/idle_electric_power.sql delete mode 100644 models/transformations/power_consumption/node_distribution.sql delete mode 100644 models/transformations/validators/consensus_validators_status.sql delete mode 100644 models/transformations/validators/test_chaind.sql create mode 100644 scripts/ymls/requirements.txt create mode 100644 scripts/ymls/ymls_generator.py diff --git a/macros/consensus/compute_timestamp_at_slot.sql b/macros/consensus/compute_timestamp_at_slot.sql new file mode 100644 index 0000000..1f20aaf --- /dev/null +++ b/macros/consensus/compute_timestamp_at_slot.sql @@ -0,0 +1,19 @@ +{% macro compute_timestamp_at_slot(slot) %} + +addSeconds( + (SELECT f_time FROM {{ get_postgres('chaind', 't_genesis') }} LIMIT 1), + ({{ slot }} - 0) * (SELECT toInt32(f_value) FROM {{ get_postgres('chaind', 't_chain_spec') }} WHERE f_key = 'SECONDS_PER_SLOT' LIMIT 1) +) + +{% endmacro %} + + +{% macro compute_timestamp_at_epoch(epoch) %} + +addSeconds( + (SELECT f_time FROM {{ get_postgres('chaind', 't_genesis') }} LIMIT 1), + ({{ epoch }} - 0) * (SELECT toInt32(f_value) FROM {{ get_postgres('chaind', 't_chain_spec') }} WHERE f_key = 'SECONDS_PER_SLOT' LIMIT 1) + *(SELECT toInt32(f_value) FROM {{ get_postgres('chaind', 't_chain_spec') }} WHERE f_key = 'SLOTS_PER_EPOCH' LIMIT 1) +) + +{% endmacro %} \ No newline at end of file diff --git a/macros/consensus/decode_graffiti.sql b/macros/consensus/decode_graffiti.sql new file mode 100644 index 0000000..674f123 --- /dev/null +++ b/macros/consensus/decode_graffiti.sql @@ -0,0 +1,11 @@ +{% macro decode_graffiti(f_graffiti) %} + -- reinterpretAsString( + unhex( + replaceRegexpOne( + {{ f_graffiti }}, + '^\\\\x', + '' + ) + ) + -- ) +{% endmacro %} diff --git a/macros/consensus/get_chain_spec.sql b/macros/consensus/get_chain_spec.sql new file mode 100644 index 0000000..1e645f1 --- /dev/null +++ b/macros/consensus/get_chain_spec.sql @@ -0,0 +1,3 @@ +{% macro get_chain_spec(spec_key) %} +(SELECT f_value FROM {{ get_postgres('chaind', 't_chain_spec') }} WHERE f_key = '{{ spec_key }}' LIMIT 1) +{% endmacro %} \ No newline at end of file diff --git a/macros/db/seconds_until_end_of_day.sql b/macros/consensus/seconds_until_end_of_day.sql similarity index 100% rename from macros/db/seconds_until_end_of_day.sql rename to macros/consensus/seconds_until_end_of_day.sql diff --git a/macros/db/compute_timestamp_at_slot.sql b/macros/db/compute_timestamp_at_slot.sql deleted file mode 100644 index 62663fe..0000000 --- a/macros/db/compute_timestamp_at_slot.sql +++ /dev/null @@ -1,19 +0,0 @@ -{% macro compute_timestamp_at_slot(slot) %} - -addSeconds( - (SELECT f_time FROM {{ get_postgres('gnosis_chaind', 't_genesis') }} LIMIT 1), - ({{ slot }} - 0) * (SELECT toInt32(f_value) FROM {{ get_postgres('gnosis_chaind', 't_chain_spec') }} WHERE f_key = 'SECONDS_PER_SLOT' LIMIT 1) -) - -{% endmacro %} - - -{% macro compute_timestamp_at_epoch(epoch) %} - -addSeconds( - (SELECT f_time FROM {{ get_postgres('gnosis_chaind', 't_genesis') }} LIMIT 1), - ({{ epoch }} - 0) * (SELECT toInt32(f_value) FROM {{ get_postgres('gnosis_chaind', 't_chain_spec') }} WHERE f_key = 'SECONDS_PER_SLOT' LIMIT 1) - *(SELECT toInt32(f_value) FROM {{ get_postgres('gnosis_chaind', 't_chain_spec') }} WHERE f_key = 'SLOTS_PER_EPOCH' LIMIT 1) -) - -{% endmacro %} \ No newline at end of file diff --git a/macros/db/get_chain_spec.sql b/macros/db/get_chain_spec.sql deleted file mode 100644 index 611de63..0000000 --- a/macros/db/get_chain_spec.sql +++ /dev/null @@ -1,3 +0,0 @@ -{% macro get_chain_spec(spec_key) %} -(SELECT f_value FROM {{ get_postgres('gnosis_chaind', 't_chain_spec') }} WHERE f_key = '{{ spec_key }}' LIMIT 1) -{% endmacro %} \ No newline at end of file diff --git a/models/consensus/attestations/metrics/consensus_atts_inclusion_distance_d.sql b/models/consensus/attestations/metrics/consensus_atts_inclusion_distance_d.sql new file mode 100644 index 0000000..4b5cd75 --- /dev/null +++ b/models/consensus/attestations/metrics/consensus_atts_inclusion_distance_d.sql @@ -0,0 +1,28 @@ +{{ + config( + materialized='incremental', + incremental_strategy='delete+insert', + engine='ReplacingMergeTree()', + order_by='(day, inc_dist_cohort)', + primary_key='(day, inc_dist_cohort)', + partition_by='partition_month' + ) +}} + +WITH + + +inclusion_distance AS ( + SELECT + toStartOfMonth({{ compute_timestamp_at_slot('f_inclusion_slot') }}) AS partition_month + ,toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) AS day + ,f_inclusion_slot - f_slot AS inc_dist_cohort + ,COUNT(*) AS cnt + FROM {{ get_postgres('chaind', 't_attestations') }} + {% if is_incremental() %} + WHERE toStartOfMonth({{ compute_timestamp_at_slot('f_inclusion_slot') }}) >= (SELECT max(partition_month) FROM {{ this }}) + {% endif %} + GROUP BY 1,2,3 +) + +SELECT * FROM inclusion_distance diff --git a/models/consensus/attestations/metrics/schema.yml b/models/consensus/attestations/metrics/schema.yml new file mode 100644 index 0000000..5d8030b --- /dev/null +++ b/models/consensus/attestations/metrics/schema.yml @@ -0,0 +1,38 @@ +version: 2 +models: +- name: consensus_atts_inclusion_distance_d + meta: + blockchain: consensus + sector: attestations + contributors: 'TODO: Add contributors' + config: + tags: + - consensus + - attestations + description: 'TODO: Add description for consensus_atts_inclusion_distance_d' + columns: + - name: cnt + description: 'TODO: Add description for cnt' + data_tests: + - not_null + - unique + - name: day + description: 'TODO: Add description for day' + data_tests: + - not_null + - unique + - name: inc_dist_cohort + description: 'TODO: Add description for inc_dist_cohort' + data_tests: + - not_null + - unique + - name: partition_month + description: 'TODO: Add description for partition_month' + data_tests: + - not_null + - unique + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - cnt + - day diff --git a/models/consensus/blocks/metrics/consensus_blocks_graffiti_top10_d.sql b/models/consensus/blocks/metrics/consensus_blocks_graffiti_top10_d.sql new file mode 100644 index 0000000..6470a90 --- /dev/null +++ b/models/consensus/blocks/metrics/consensus_blocks_graffiti_top10_d.sql @@ -0,0 +1,62 @@ +{{ + config( + materialized='incremental', + incremental_strategy='delete+insert', + engine='ReplacingMergeTree()', + order_by='(day, graffiti_top10)', + primary_key='(day, graffiti_top10)', + partition_by='partition_month', + settings={ + "allow_nullable_key": 1 + } + ) +}} + +WITH blocks_graffiti AS ( + SELECT + partition_month + ,day + ,CASE + WHEN graffiti IS NULL + OR graffiti = '' + OR LENGTH(TRIM(graffiti)) = 0 + OR graffiti = '\0' + OR graffiti LIKE '\0%' + OR char_length(graffiti) = 0 + THEN 'No Graffiti' + WHEN POSITION('lighthouse' IN LOWER(graffiti)) > 0 THEN 'Lighthouse' + WHEN POSITION('nimbus' IN LOWER(graffiti)) > 0 THEN 'Nimbus' + WHEN POSITION('prysm' IN LOWER(graffiti)) > 0 THEN 'Prysm' + WHEN POSITION('teku' IN LOWER(graffiti)) > 0 THEN 'Teku' + WHEN POSITION('dappnode' IN LOWER(graffiti)) > 0 THEN 'DappNode' + WHEN POSITION('stakewise' IN LOWER(graffiti)) > 0 THEN 'StateWise' + ELSE graffiti + END AS graffiti + ,cnt + ,ROW_NUMBER() OVER (PARTITION BY day ORDER BY cnt DESC) AS r_cnt + FROM ( + SELECT + partition_month + ,day + ,graffiti + ,SUM(cnt) AS cnt + FROM + {{ ref('consensus_blocks_graffiti') }} + {% if is_incremental() %} + WHERE partition_month >= (SELECT max(partition_month) FROM {{ this }}) + {% endif %} + GROUP BY 1, 2, 3 + ) + +) + +SELECT + partition_month + ,day + ,CASE + WHEN r_cnt <= 10 THEN graffiti + ELSE 'Others' + END AS graffiti_top10 + ,SUM(cnt) AS cnt +FROM blocks_graffiti +GROUP BY 1, 2, 3 \ No newline at end of file diff --git a/models/consensus/blocks/metrics/consensus_blocks_graffiti_validators.sql b/models/consensus/blocks/metrics/consensus_blocks_graffiti_validators.sql new file mode 100644 index 0000000..86b3acf --- /dev/null +++ b/models/consensus/blocks/metrics/consensus_blocks_graffiti_validators.sql @@ -0,0 +1,119 @@ +{{ config(materialized='table') }} + +WITH + +-- Get the withdrawal time for validators +withdrawable_validators AS ( + SELECT + f_index AS proposer_index, + {{ compute_timestamp_at_epoch('f_withdrawable_epoch') }} AS withdrawable_time + FROM {{ get_postgres('gnosis_chaind', 't_validators') }} + WHERE f_withdrawable_epoch IS NOT NULL +), + +-- Retrieve classified graffiti per proposer per day +daily_graffiti AS ( + SELECT + day, + f_proposer_index, + CASE + WHEN graffiti IS NULL + OR graffiti = '' + OR LENGTH(TRIM(graffiti)) = 0 + OR graffiti = '\0' + OR graffiti LIKE '\0%' + OR char_length(graffiti) = 0 + THEN 'No Graffiti' + WHEN POSITION('lighthouse' IN LOWER(graffiti)) > 0 THEN 'Lighthouse' + WHEN POSITION('nimbus' IN LOWER(graffiti)) > 0 THEN 'Nimbus' + WHEN POSITION('prysm' IN LOWER(graffiti)) > 0 THEN 'Prysm' + WHEN POSITION('teku' IN LOWER(graffiti)) > 0 THEN 'Teku' + WHEN POSITION('dappnode' IN LOWER(graffiti)) > 0 THEN 'DappNode' + WHEN POSITION('stakewise' IN LOWER(graffiti)) > 0 THEN 'StakeWise' + ELSE graffiti + END AS graffiti + FROM + {{ ref('consensus_blocks_graffiti') }} +), + +-- Assign row numbers to identify sequences of the same graffiti +graffiti_changes AS ( + SELECT + f_proposer_index, + day, + graffiti, + ROW_NUMBER() OVER (PARTITION BY f_proposer_index ORDER BY day) AS rn1, + ROW_NUMBER() OVER (PARTITION BY f_proposer_index, graffiti ORDER BY day) AS rn2 + FROM daily_graffiti +), + +-- Calculate group identifiers based on row number differences +graffiti_periods AS ( + SELECT + f_proposer_index, + graffiti, + (rn1 - rn2) AS grp, + MIN(day) AS start_day, + MAX(day) AS end_day + FROM graffiti_changes + GROUP BY f_proposer_index, graffiti, grp +), + +-- Assign proposers to their graffiti on each day within their graffiti periods +proposer_graffiti_on_day AS ( + SELECT + gp.f_proposer_index, + day, + gp.graffiti + FROM graffiti_periods gp + ARRAY JOIN + arrayMap(i -> toDate(gp.start_day + i), range(toUInt32(gp.end_day - gp.start_day) + 1)) AS day +), + +-- Exclude withdrawable proposers on and after their withdrawal date +active_proposers AS ( + SELECT + pgod.f_proposer_index, + pgod.day, + pgod.graffiti + FROM proposer_graffiti_on_day pgod + LEFT JOIN withdrawable_validators wv ON pgod.f_proposer_index = wv.proposer_index + WHERE wv.withdrawable_time IS NULL OR wv.withdrawable_time > pgod.day +), + +-- Count unique active proposers per day and graffiti +cumulative_counts AS ( + SELECT + day, + graffiti AS graffiti_type, + COUNT(DISTINCT f_proposer_index) AS cnt + FROM active_proposers + GROUP BY day, graffiti +), + +-- Rank graffiti types by proposer count per day +ranked_graffiti AS ( + SELECT + day, + graffiti_type, + cnt, + ROW_NUMBER() OVER (PARTITION BY day ORDER BY cnt DESC) AS r_cnt + FROM cumulative_counts +) + +-- Final aggregation and labeling of top 10 graffiti types +SELECT + * +FROM ( + SELECT + day, + CASE + WHEN r_cnt <= 10 THEN graffiti_type + ELSE 'Others' + END AS graffiti_top10, + SUM(cnt) AS cnt + FROM ranked_graffiti + GROUP BY 1, 2 + ORDER BY day, graffiti_top10 +) +WHERE cnt>0 diff --git a/models/transformations/attestations/consensus_blocks_production.sql b/models/consensus/blocks/metrics/consensus_blocks_production.sql similarity index 75% rename from models/transformations/attestations/consensus_blocks_production.sql rename to models/consensus/blocks/metrics/consensus_blocks_production.sql index 7665c71..ee04044 100644 --- a/models/transformations/attestations/consensus_blocks_production.sql +++ b/models/consensus/blocks/metrics/consensus_blocks_production.sql @@ -2,25 +2,25 @@ config( materialized='incremental', incremental_strategy='insert_overwrite', - partition_by='day' + partition_by='toStartOfWeek(day)' ) }} WITH genesis AS ( SELECT f_time AS genesis_time - FROM {{ get_postgres('gnosis_chaind', 't_genesis') }} + FROM {{ get_postgres('chaind', 't_genesis') }} LIMIT 1 ), blocks AS ( SELECT toDate({{ compute_timestamp_at_slot('f_slot') }}) AS day, MIN(f_slot) AS f_slot_start, - CAST(SUM(IF(NOT f_canonical, 1, 0)) AS Int64) AS forked, - CAST(SUM(IF(f_canonical IS NULL, 1, 0)) AS Int64) AS proposed + CAST(SUM(IF(f_canonical, 0, 1)) AS Int64) AS forked, + CAST(SUM(IF(f_canonical, 1, 0)) AS Int64) AS proposed FROM - {{ get_postgres('gnosis_chaind', 't_blocks') }} + {{ get_postgres('chaind', 't_blocks') }} {% if is_incremental() %} - WHERE toDate({{ compute_timestamp_at_slot('f_slot') }}) >= (SELECT max(day) FROM {{ this }}) + WHERE toStartOfWeek(toDate({{ compute_timestamp_at_slot('f_slot') }})) >= (SELECT max(toStartOfWeek(day)) FROM {{ this }}) {% endif %} GROUP BY 1 ), diff --git a/models/consensus/blocks/metrics/schema.yml b/models/consensus/blocks/metrics/schema.yml new file mode 100644 index 0000000..5d5a783 --- /dev/null +++ b/models/consensus/blocks/metrics/schema.yml @@ -0,0 +1,85 @@ +version: 2 +models: +- name: consensus_blocks_graffiti_validators + meta: + blockchain: consensus + sector: blocks + contributors: 'TODO: Add contributors' + config: + tags: + - consensus + - blocks + description: 'TODO: Add description for consensus_blocks_graffiti_validators' + columns: + - name: proposer_index + description: 'TODO: Add description for proposer_index' + data_tests: + - not_null + - unique + - name: withdrawable_time + description: 'TODO: Add description for withdrawable_time' + data_tests: + - not_null + - unique + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - proposer_index + - withdrawable_time +- name: consensus_blocks_graffiti_top10_d + meta: + blockchain: consensus + sector: blocks + contributors: 'TODO: Add contributors' + config: + tags: + - consensus + - blocks + description: 'TODO: Add description for consensus_blocks_graffiti_top10_d' + columns: + - name: cnt + description: 'TODO: Add description for cnt' + data_tests: + - not_null + - unique + - name: day + description: 'TODO: Add description for day' + data_tests: + - not_null + - unique + - name: graffiti + description: 'TODO: Add description for graffiti' + data_tests: + - not_null + - unique + - name: partition_month + description: 'TODO: Add description for partition_month' + data_tests: + - not_null + - unique + - name: r_cnt + description: 'TODO: Add description for r_cnt' + data_tests: + - not_null + - unique + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - cnt + - day +- name: consensus_blocks_production + meta: + blockchain: consensus + sector: blocks + contributors: 'TODO: Add contributors' + config: + tags: + - consensus + - blocks + description: 'TODO: Add description for consensus_blocks_production' + columns: + - name: genesis_time + description: 'TODO: Add description for genesis_time' + data_tests: + - not_null + - unique diff --git a/models/consensus/blocks/transformations/consensus_blocks_graffiti.sql b/models/consensus/blocks/transformations/consensus_blocks_graffiti.sql new file mode 100644 index 0000000..99d22fb --- /dev/null +++ b/models/consensus/blocks/transformations/consensus_blocks_graffiti.sql @@ -0,0 +1,28 @@ +{{ + config( + materialized='incremental', + incremental_strategy='delete+insert', + engine='ReplacingMergeTree()', + order_by='(day, graffiti, f_proposer_index)', + primary_key='(day, graffiti, f_proposer_index)', + partition_by='partition_month' + ) +}} + + +WITH blocks_graffiti AS ( + SELECT + toStartOfMonth({{ compute_timestamp_at_slot('f_slot') }}) AS partition_month + ,toDate({{ compute_timestamp_at_slot('f_slot') }}) AS day + ,f_proposer_index + ,{{ decode_graffiti('f_graffiti') }} AS graffiti + ,COUNT(*) AS cnt + FROM + {{ get_postgres('chaind', 't_blocks') }} + {% if is_incremental() %} + WHERE toStartOfMonth({{ compute_timestamp_at_slot('f_slot') }}) >= (SELECT max(partition_month) FROM {{ this }}) + {% endif %} + GROUP BY 1, 2, 3, 4 +) + +SELECT * FROM blocks_graffiti diff --git a/models/consensus/blocks/transformations/schema.yml b/models/consensus/blocks/transformations/schema.yml new file mode 100644 index 0000000..41eaa85 --- /dev/null +++ b/models/consensus/blocks/transformations/schema.yml @@ -0,0 +1,43 @@ +version: 2 +models: +- name: consensus_blocks_graffiti + meta: + blockchain: consensus + sector: blocks + contributors: 'TODO: Add contributors' + config: + tags: + - consensus + - blocks + description: 'TODO: Add description for consensus_blocks_graffiti' + columns: + - name: cnt + description: 'TODO: Add description for cnt' + data_tests: + - not_null + - unique + - name: day + description: 'TODO: Add description for day' + data_tests: + - not_null + - unique + - name: f_proposer_index + description: 'TODO: Add description for f_proposer_index' + data_tests: + - not_null + - unique + - name: graffiti + description: 'TODO: Add description for graffiti' + data_tests: + - not_null + - unique + - name: partition_month + description: 'TODO: Add description for partition_month' + data_tests: + - not_null + - unique + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - cnt + - day diff --git a/models/consensus/consensus_sources.yml b/models/consensus/consensus_sources.yml new file mode 100644 index 0000000..80cc9a8 --- /dev/null +++ b/models/consensus/consensus_sources.yml @@ -0,0 +1,30 @@ +sources: + - name: chaind + database: chaind + schema: public + tables: + - name: t_block_execution_payloads + - name: t_block_withdrawals + - name: t_deposits + - name: t_attestations + - name: t_attester_slashings + - name: t_beacon_committees + - name: t_blob_sidecars + - name: t_block_bls_to_execution_changes + - name: t_block_summaries + - name: t_blocks + - name: t_chain_spec + - name: t_epoch_summaries + - name: t_eth1_deposits + - name: t_genesis + - name: t_metadata + - name: t_proposer_duties + - name: t_fork_schedule + - name: t_proposer_slashings + - name: t_sync_aggregates + - name: t_sync_committees + - name: t_validator_balances + - name: t_validator_day_summaries + - name: t_validator_epoch_summaries + - name: t_validators + - name: t_voluntary_exits diff --git a/models/metrics/validators/consensus_validators_activations.sql b/models/consensus/validators/metrics/consensus_validators_activations.sql similarity index 84% rename from models/metrics/validators/consensus_validators_activations.sql rename to models/consensus/validators/metrics/consensus_validators_activations.sql index 5662e91..d6d256c 100644 --- a/models/metrics/validators/consensus_validators_activations.sql +++ b/models/consensus/validators/metrics/consensus_validators_activations.sql @@ -12,7 +12,7 @@ WITH validators_activations AS ( toStartOfMonth(toDate(activation_time)) AS month, CAST(COUNT(*) AS Int64) AS cnt, 'activations' AS label - FROM {{ ref('consensus_validators_status') }} + FROM {{ ref('consensus_validators_queue') }} {% if is_incremental() %} WHERE toDate(activation_time) >= (SELECT max(day) FROM {{ this }}) {% endif %} @@ -21,11 +21,11 @@ WITH validators_activations AS ( validators_exits AS ( SELECT - toDate(withdrawable_time) AS day, - toStartOfMonth(toDate(withdrawable_time)) AS month, + toDate(exit_time) AS day, + toStartOfMonth(toDate(exit_time)) AS month, -CAST(COUNT(*) AS Int64) AS cnt, 'exits' AS label - FROM {{ ref('consensus_validators_status') }} + FROM {{ ref('consensus_validators_queue') }} WHERE exit_time IS NOT NULL {% if is_incremental() %} diff --git a/models/consensus/validators/metrics/consensus_validators_eff_balance_dist.sql b/models/consensus/validators/metrics/consensus_validators_eff_balance_dist.sql new file mode 100644 index 0000000..f159189 --- /dev/null +++ b/models/consensus/validators/metrics/consensus_validators_eff_balance_dist.sql @@ -0,0 +1,44 @@ +{{ + config( + materialized='table', + ) +}} + +{% set num_bins = 100 %} + +WITH stats AS ( + SELECT + MIN(CAST(f_effective_balance AS FLOAT)/32000000000) as min_total, + MAX(CAST(f_effective_balance AS FLOAT)/32000000000) as max_total, + (MAX(CAST(f_effective_balance AS FLOAT)/32000000000) - MIN(CAST(f_effective_balance AS FLOAT)/32000000000))/{{num_bins}} as bin_size + FROM {{ get_postgres('gnosis_chaind', 't_validators') }} +), +series AS ( + SELECT number + 1 as bucket + FROM numbers({{num_bins}}) +), +validators_effective_balance AS ( + SELECT + width_bucket( + CAST(f_effective_balance AS FLOAT)/32000000000, + (SELECT min_total FROM stats), + (SELECT max_total FROM stats) + 0.000001, -- Add small amount to include upper bound + {{num_bins}} + ) as bucket, + COUNT(*) as cnt + FROM {{ get_postgres('gnosis_chaind', 't_validators') }} + GROUP BY 1 +) + +SELECT + s.bucket, + CONCAT( + ROUND(min_total + (s.bucket-1) * bin_size, 2), + ' - ', + ROUND(min_total + s.bucket * bin_size, 2) + ) as range, + COALESCE(v.cnt, 0) as cnt +FROM series s +CROSS JOIN stats +LEFT JOIN validators_effective_balance v ON s.bucket = v.bucket +ORDER BY s.bucket \ No newline at end of file diff --git a/models/consensus/validators/metrics/consensus_validators_entry_queue.sql b/models/consensus/validators/metrics/consensus_validators_entry_queue.sql new file mode 100644 index 0000000..b5c23b7 --- /dev/null +++ b/models/consensus/validators/metrics/consensus_validators_entry_queue.sql @@ -0,0 +1,86 @@ +{{ + config( + materialized='incremental', + incremental_strategy='insert_overwrite', + partition_by='partition_month', + settings={ + "allow_nullable_key": 1 + } + ) +}} + +WITH + +{% if is_incremental() %} +last_partition AS ( + SELECT max(partition_month) as partition_month + FROM {{ this }} +), +{% endif %} + +validators_in_activation_queue AS ( + SELECT + toDate(f_eth1_block_timestamp) AS day, + toStartOfMonth(toDate(f_eth1_block_timestamp)) AS partition_month, + CAST(COUNT(*) AS Int64) AS cnt + FROM {{ ref('consensus_validators_queue') }} + {% if is_incremental() %} + WHERE toStartOfMonth(toDate(f_eth1_block_timestamp)) >= (SELECT partition_month FROM last_partition) + {% endif %} + GROUP BY 1, 2 +), + +validators_activated AS ( + SELECT + toDate(activation_time) AS day, + toStartOfMonth(toDate(activation_time)) AS partition_month, + CAST(COUNT(*) AS Int64) AS cnt + FROM {{ ref('consensus_validators_queue') }} + {% if is_incremental() %} + WHERE toStartOfMonth(toDate(f_eth1_block_timestamp)) >= (SELECT partition_month FROM last_partition) + {% endif %} + GROUP BY 1, 2 +), + +min_max_dates AS ( + SELECT + min(dates.day) as min_date, + max(dates.day) as max_date + FROM ( + SELECT day FROM validators_in_activation_queue + UNION ALL + SELECT day FROM validators_activated + ) dates +), + +calendar AS ( + SELECT + toDate(min_date + number) AS day, + toStartOfMonth(toDate(min_date + number)) AS partition_month + FROM min_max_dates + CROSS JOIN ( + SELECT arrayJoin(range(dateDiff('day', min_date, max_date) + 1)) AS number + FROM min_max_dates + ) s +), + +daily_metrics AS ( + SELECT + c.day AS day, + c.partition_month As partition_month, + COALESCE(q.cnt, 0) AS validators_entered_queue, + COALESCE(a.cnt, 0) AS validators_activated, + COALESCE(q.cnt, 0) - COALESCE(a.cnt, 0) AS net_queue_change + FROM calendar c + LEFT JOIN validators_in_activation_queue q ON c.day = q.day + LEFT JOIN validators_activated a ON c.day = a.day +) + + +SELECT + day, + partition_month, + validators_entered_queue, + validators_activated, + net_queue_change +FROM daily_metrics \ No newline at end of file diff --git a/models/consensus/validators/metrics/consensus_validators_participation_rate.sql b/models/consensus/validators/metrics/consensus_validators_participation_rate.sql new file mode 100644 index 0000000..4ae1131 --- /dev/null +++ b/models/consensus/validators/metrics/consensus_validators_participation_rate.sql @@ -0,0 +1,31 @@ +{{ + config( + materialized='incremental', + incremental_strategy='delete+insert', + engine='ReplacingMergeTree()', + order_by='(day)', + primary_key='(day)', + partition_by='partition_month' + ) +}} + +WITH committe_size AS ( + SELECT + CAST(f_value AS FLOAT) AS f_value + FROM + {{ get_postgres('chaind', 't_chain_spec') }} + WHERE + f_key = 'EPOCHS_PER_SYNC_COMMITTEE_PERIOD' +) + +SELECT + toStartOfMonth(toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }})) AS partition_month + ,toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) AS day + ,AVG(length(f_indices)/(SELECT f_value FROM committe_size)) AS mean_pct +FROM + {{ get_postgres('chaind', 't_sync_aggregates') }} +{% if is_incremental() %} +WHERE toStartOfMonth(toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }})) >= (SELECT max(partition_month) FROM {{ this }}) +{% endif %} +GROUP BY 1, 2 + diff --git a/models/consensus/validators/metrics/consensus_validators_queue_cnt_d.sql b/models/consensus/validators/metrics/consensus_validators_queue_cnt_d.sql new file mode 100644 index 0000000..fb9647a --- /dev/null +++ b/models/consensus/validators/metrics/consensus_validators_queue_cnt_d.sql @@ -0,0 +1,56 @@ +{{ + config( + materialized='table', + ) +}} + +{% set delay_fields = [ + {'field': 'eligibility_delay', 'label': 'Eligibility'}, + {'field': 'activation_delay', 'label': 'Activation'}, + {'field': 'entry_delay', 'label': 'Entry'}, + {'field': 'exit_activation_delay', 'label': 'De-activation'}, + {'field': 'exit_withdrawable_delay', 'label': 'Withdrawable'}, + {'field': 'queue_exit_delay', 'label': 'Exit'}, + {'field': 'exit_delay', 'label': 'Exit Delay'} +] %} + +{% set aggregations = [ + {'func': 'min', 'label': 'Min'}, + {'func': 'max', 'label': 'Max'}, + {'func': 'median', 'label': 'Median'}, + {'func': 'avg', 'label': 'Mean'} +] %} + +WITH final AS ( + SELECT + f_validator_pubkey, + MIN(f_eth1_block_timestamp) AS eth1_block_timestamp, + MAX(toInt64(activation_eligibility_time) - toInt64(f_eth1_block_timestamp)) AS eligibility_delay, + MAX(toInt64(activation_time) - toInt64(activation_eligibility_time)) AS activation_delay, + MAX(toInt64(activation_time) - toInt64(f_eth1_block_timestamp)) AS entry_delay, + MAX(toInt64(exit_time) - toInt64(exit_request_time)) AS exit_activation_delay, + MAX(toInt64(withdrawable_time) - toInt64(exit_time)) AS exit_withdrawable_delay, + MAX(toInt64(withdrawable_time) - toInt64(exit_request_time)) AS queue_exit_delay, + MAX(toInt64(withdrawable_time) - toInt64(exit_voluntary_time)) AS exit_delay + FROM + {{ ref('consensus_validators_queue') }} + GROUP BY + f_validator_pubkey +) + +{% set union_queries = [] %} +{% for delay in delay_fields %} + {% for agg in aggregations %} + {% set query %} +SELECT + toDate(eth1_block_timestamp) AS day, + COALESCE(toFloat64({{ agg.func }}({{ delay.field }})), 0) / 3600 AS value, + '{{ agg.label }} {{ delay.label }}' AS label +FROM final +GROUP BY day + {% endset %} + {% do union_queries.append(query) %} + {% endfor %} +{% endfor %} + +{{ union_queries | join('\nUNION ALL\n') }} \ No newline at end of file diff --git a/models/consensus/validators/metrics/consensus_validators_waiting_times.sql b/models/consensus/validators/metrics/consensus_validators_waiting_times.sql new file mode 100644 index 0000000..fb9647a --- /dev/null +++ b/models/consensus/validators/metrics/consensus_validators_waiting_times.sql @@ -0,0 +1,56 @@ +{{ + config( + materialized='table', + ) +}} + +{% set delay_fields = [ + {'field': 'eligibility_delay', 'label': 'Eligibility'}, + {'field': 'activation_delay', 'label': 'Activation'}, + {'field': 'entry_delay', 'label': 'Entry'}, + {'field': 'exit_activation_delay', 'label': 'De-activation'}, + {'field': 'exit_withdrawable_delay', 'label': 'Withdrawable'}, + {'field': 'queue_exit_delay', 'label': 'Exit'}, + {'field': 'exit_delay', 'label': 'Exit Delay'} +] %} + +{% set aggregations = [ + {'func': 'min', 'label': 'Min'}, + {'func': 'max', 'label': 'Max'}, + {'func': 'median', 'label': 'Median'}, + {'func': 'avg', 'label': 'Mean'} +] %} + +WITH final AS ( + SELECT + f_validator_pubkey, + MIN(f_eth1_block_timestamp) AS eth1_block_timestamp, + MAX(toInt64(activation_eligibility_time) - toInt64(f_eth1_block_timestamp)) AS eligibility_delay, + MAX(toInt64(activation_time) - toInt64(activation_eligibility_time)) AS activation_delay, + MAX(toInt64(activation_time) - toInt64(f_eth1_block_timestamp)) AS entry_delay, + MAX(toInt64(exit_time) - toInt64(exit_request_time)) AS exit_activation_delay, + MAX(toInt64(withdrawable_time) - toInt64(exit_time)) AS exit_withdrawable_delay, + MAX(toInt64(withdrawable_time) - toInt64(exit_request_time)) AS queue_exit_delay, + MAX(toInt64(withdrawable_time) - toInt64(exit_voluntary_time)) AS exit_delay + FROM + {{ ref('consensus_validators_queue') }} + GROUP BY + f_validator_pubkey +) + +{% set union_queries = [] %} +{% for delay in delay_fields %} + {% for agg in aggregations %} + {% set query %} +SELECT + toDate(eth1_block_timestamp) AS day, + COALESCE(toFloat64({{ agg.func }}({{ delay.field }})), 0) / 3600 AS value, + '{{ agg.label }} {{ delay.label }}' AS label +FROM final +GROUP BY day + {% endset %} + {% do union_queries.append(query) %} + {% endfor %} +{% endfor %} + +{{ union_queries | join('\nUNION ALL\n') }} \ No newline at end of file diff --git a/models/consensus/validators/metrics/schema.yml b/models/consensus/validators/metrics/schema.yml new file mode 100644 index 0000000..869350b --- /dev/null +++ b/models/consensus/validators/metrics/schema.yml @@ -0,0 +1,223 @@ +version: 2 +models: +- name: consensus_validators_waiting_times + meta: + blockchain: consensus + sector: validators + contributors: 'TODO: Add contributors' + config: + tags: + - consensus + - validators + description: 'TODO: Add description for consensus_validators_waiting_times' + columns: + - name: activation_delay + description: 'TODO: Add description for activation_delay' + data_tests: + - not_null + - unique + - name: eligibility_delay + description: 'TODO: Add description for eligibility_delay' + data_tests: + - not_null + - unique + - name: entry_delay + description: 'TODO: Add description for entry_delay' + data_tests: + - not_null + - unique + - name: eth1_block_timestamp + description: 'TODO: Add description for eth1_block_timestamp' + data_tests: + - not_null + - unique + - name: exit_activation_delay + description: 'TODO: Add description for exit_activation_delay' + data_tests: + - not_null + - unique + - name: exit_delay + description: 'TODO: Add description for exit_delay' + data_tests: + - not_null + - unique + - name: exit_withdrawable_delay + description: 'TODO: Add description for exit_withdrawable_delay' + data_tests: + - not_null + - unique + - name: f_validator_pubkey + description: 'TODO: Add description for f_validator_pubkey' + data_tests: + - not_null + - unique + - name: queue_exit_delay + description: 'TODO: Add description for queue_exit_delay' + data_tests: + - not_null + - unique + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - activation_delay + - eligibility_delay +- name: consensus_validators_eff_balance_dist + meta: + blockchain: consensus + sector: validators + contributors: 'TODO: Add contributors' + config: + tags: + - consensus + - validators + description: 'TODO: Add description for consensus_validators_eff_balance_dist' + columns: + - name: FLOAT)/32000000000) as max_total + description: 'TODO: Add description for FLOAT)/32000000000) as max_total' + data_tests: + - not_null + - unique + - name: FLOAT)/32000000000) as min_total + description: 'TODO: Add description for FLOAT)/32000000000) as min_total' + data_tests: + - not_null + - unique + - name: FLOAT)/32000000000))/{{num_bins}} as bin_size + description: 'TODO: Add description for FLOAT)/32000000000))/{{num_bins}} as bin_size' + data_tests: + - not_null + - unique + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - FLOAT)/32000000000) as max_total + - FLOAT)/32000000000) as min_total +- name: consensus_validators_participation_rate + meta: + blockchain: consensus + sector: validators + contributors: 'TODO: Add contributors' + config: + tags: + - consensus + - validators + description: 'TODO: Add description for consensus_validators_participation_rate' + columns: + - name: f_value + description: 'TODO: Add description for f_value' + data_tests: + - not_null + - unique +- name: consensus_validators_activations + meta: + blockchain: consensus + sector: validators + contributors: 'TODO: Add contributors' + config: + tags: + - consensus + - validators + description: 'TODO: Add description for consensus_validators_activations' + columns: + - name: cnt + description: 'TODO: Add description for cnt' + data_tests: + - not_null + - unique + - name: day + description: 'TODO: Add description for day' + data_tests: + - not_null + - unique + - name: label + description: 'TODO: Add description for label' + data_tests: + - not_null + - unique + - name: month + description: 'TODO: Add description for month' + data_tests: + - not_null + - unique + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - cnt + - day +- name: consensus_validators_entry_queue + meta: + blockchain: consensus + sector: validators + contributors: 'TODO: Add contributors' + config: + tags: + - consensus + - validators + description: 'TODO: Add description for consensus_validators_entry_queue' + columns: + - name: max(partition_month) as partition_month + description: 'TODO: Add description for max(partition_month) as partition_month' + data_tests: + - not_null + - unique +- name: consensus_validators_queue_cnt_d + meta: + blockchain: consensus + sector: validators + contributors: 'TODO: Add contributors' + config: + tags: + - consensus + - validators + description: 'TODO: Add description for consensus_validators_queue_cnt_d' + columns: + - name: activation_delay + description: 'TODO: Add description for activation_delay' + data_tests: + - not_null + - unique + - name: eligibility_delay + description: 'TODO: Add description for eligibility_delay' + data_tests: + - not_null + - unique + - name: entry_delay + description: 'TODO: Add description for entry_delay' + data_tests: + - not_null + - unique + - name: eth1_block_timestamp + description: 'TODO: Add description for eth1_block_timestamp' + data_tests: + - not_null + - unique + - name: exit_activation_delay + description: 'TODO: Add description for exit_activation_delay' + data_tests: + - not_null + - unique + - name: exit_delay + description: 'TODO: Add description for exit_delay' + data_tests: + - not_null + - unique + - name: exit_withdrawable_delay + description: 'TODO: Add description for exit_withdrawable_delay' + data_tests: + - not_null + - unique + - name: f_validator_pubkey + description: 'TODO: Add description for f_validator_pubkey' + data_tests: + - not_null + - unique + - name: queue_exit_delay + description: 'TODO: Add description for queue_exit_delay' + data_tests: + - not_null + - unique + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - activation_delay + - eligibility_delay diff --git a/models/consensus/validators/transformations/consensus_validators_queue.sql b/models/consensus/validators/transformations/consensus_validators_queue.sql new file mode 100644 index 0000000..15180fb --- /dev/null +++ b/models/consensus/validators/transformations/consensus_validators_queue.sql @@ -0,0 +1,45 @@ +{{ + config( + materialized='incremental', + incremental_strategy='delete+insert', + engine='ReplacingMergeTree()', + order_by='(f_eth1_block_timestamp, f_validator_pubkey)', + primary_key='(f_eth1_block_timestamp, f_validator_pubkey)', + partition_by='partition_month' + ) +}} + +SELECT + toStartOfMonth(el.f_eth1_block_timestamp) AS partition_month + ,v.f_index + ,el.f_validator_pubkey AS f_validator_pubkey + ,el.f_withdrawal_credentials + ,el.f_signature + ,el.f_eth1_block_timestamp AS f_eth1_block_timestamp + ,el.f_eth1_gas_used + ,el.f_eth1_gas_price + ,el.f_amount + ,{{ compute_timestamp_at_slot('cl.f_inclusion_slot') }} AS inclusion_time + ,{{ compute_timestamp_at_epoch('v.f_activation_eligibility_epoch') }} AS activation_eligibility_time + ,{{ compute_timestamp_at_epoch('v.f_activation_epoch') }} AS activation_time + ,{{ compute_timestamp_at_slot('ve.f_inclusion_slot') }} AS exit_request_time + ,{{ compute_timestamp_at_epoch('ve.f_epoch') }} AS exit_voluntary_time + ,{{ compute_timestamp_at_epoch('v.f_exit_epoch') }} AS exit_time + ,{{ compute_timestamp_at_epoch('v.f_withdrawable_epoch') }} AS withdrawable_time +FROM + {{ get_postgres('chaind', 't_eth1_deposits') }} el +LEFT JOIN + {{ get_postgres('chaind', 't_deposits') }} cl + ON cl.f_validator_pubkey = el.f_validator_pubkey +INNER JOIN + {{ get_postgres('chaind', 't_validators') }} v + ON v.f_public_key = el.f_validator_pubkey +LEFT JOIN + {{ get_postgres('chaind', 't_voluntary_exits') }} ve + ON ve.f_validator_index = v.f_index +{% if is_incremental() %} +WHERE + toStartOfWeek(el.f_eth1_block_timestamp) >= (SELECT max(partition_week) FROM {{ this }}) +{% endif %} +SETTINGS + join_use_nulls = 1 \ No newline at end of file diff --git a/models/consensus/validators/transformations/schema.yml b/models/consensus/validators/transformations/schema.yml new file mode 100644 index 0000000..2091392 --- /dev/null +++ b/models/consensus/validators/transformations/schema.yml @@ -0,0 +1,98 @@ +version: 2 +models: +- name: consensus_validators_queue + meta: + blockchain: consensus + sector: validators + contributors: 'TODO: Add contributors' + config: + tags: + - consensus + - validators + description: 'TODO: Add description for consensus_validators_queue' + columns: + - name: activation_eligibility_time + description: 'TODO: Add description for activation_eligibility_time' + data_tests: + - not_null + - unique + - name: activation_time + description: 'TODO: Add description for activation_time' + data_tests: + - not_null + - unique + - name: exit_request_time + description: 'TODO: Add description for exit_request_time' + data_tests: + - not_null + - unique + - name: exit_time + description: 'TODO: Add description for exit_time' + data_tests: + - not_null + - unique + - name: exit_voluntary_time + description: 'TODO: Add description for exit_voluntary_time' + data_tests: + - not_null + - unique + - name: f_amount + description: 'TODO: Add description for f_amount' + data_tests: + - not_null + - unique + - name: f_eth1_block_timestamp + description: 'TODO: Add description for f_eth1_block_timestamp' + data_tests: + - not_null + - unique + - name: f_eth1_gas_price + description: 'TODO: Add description for f_eth1_gas_price' + data_tests: + - not_null + - unique + - name: f_eth1_gas_used + description: 'TODO: Add description for f_eth1_gas_used' + data_tests: + - not_null + - unique + - name: f_index + description: 'TODO: Add description for f_index' + data_tests: + - not_null + - unique + - name: f_signature + description: 'TODO: Add description for f_signature' + data_tests: + - not_null + - unique + - name: f_validator_pubkey + description: 'TODO: Add description for f_validator_pubkey' + data_tests: + - not_null + - unique + - name: f_withdrawal_credentials + description: 'TODO: Add description for f_withdrawal_credentials' + data_tests: + - not_null + - unique + - name: inclusion_time + description: 'TODO: Add description for inclusion_time' + data_tests: + - not_null + - unique + - name: partition_month + description: 'TODO: Add description for partition_month' + data_tests: + - not_null + - unique + - name: withdrawable_time + description: 'TODO: Add description for withdrawable_time' + data_tests: + - not_null + - unique + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - activation_eligibility_time + - activation_time diff --git a/models/metrics/_carbon_ratings/gnosis_carbon_emissions.sql b/models/metrics/_carbon_ratings/gnosis_carbon_emissions.sql deleted file mode 100644 index 087fb6a..0000000 --- a/models/metrics/_carbon_ratings/gnosis_carbon_emissions.sql +++ /dev/null @@ -1,51 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - - -WITH - -gnosis_power_consumption AS ( - SELECT - hour - ,country_code - ,power - FROM - {{ ref('gnosis_power_consumption') }} -), - -ember_data AS ( - SELECT - "Date" AS month_date - ,"Value" AS value - ,LAG("Value") OVER (PARTITION BY "Country_code" ORDER BY "Date") AS lag_value - ,"Country_code" AS country_code - FROM - {{ ref('ember_electricity_data') }} - WHERE - "Unit" = 'gCO2/kWh' - -) - -SELECT - t1.hour - ,SUM(t1.power * 1) AS energy - ,SUM(t1.power * 1 * COALESCE(t3.value,t3.lag_value)) AS co2_emissions - ,AVG(COALESCE(t3.value,t3.lag_value)) AS mean_cif -FROM - gnosis_power_consumption t1 -LEFT JOIN - {{ ref('country_codes') }} t2 - ON - t2."alpha-2" = t1.country_code -LEFT JOIN - ember_data t3 - ON - t3.country_code = t2."alpha-3" - AND - t3.month_date = DATE_TRUNC('month', t1.hour) -GROUP BY - t1.hour - diff --git a/models/metrics/_carbon_ratings/gnosis_power_consumption.sql b/models/metrics/_carbon_ratings/gnosis_power_consumption.sql deleted file mode 100644 index 0ebebbc..0000000 --- a/models/metrics/_carbon_ratings/gnosis_power_consumption.sql +++ /dev/null @@ -1,113 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - - -WITH - -consensus_power AS ( - SELECT - type - ,client - ,mean - FROM - {{ ref('consensus_power') }} -), - -execution_power AS ( - SELECT - type - ,client - ,mean - FROM - {{ ref('execution_power') }} -), - -idle_electric_power AS ( - SELECT - type - ,mean - FROM - {{ ref('idle_electric_power') }} -), - -node_distribution AS ( - SELECT - type - ,distribution - FROM - {{ ref('node_distribution') }} -), - -node_config_power AS ( - SELECT - t1.type - ,t1.client AS consensus_client - ,t2.client AS execution_client - ,t1.mean + t2.mean + t3.mean AS mean - FROM - consensus_power t1 - INNER JOIN - execution_power t2 - ON - t2.type = t1.type - INNER JOIN - idle_electric_power t3 - ON - t3.type = t1.type - -), - -best_guess_per_client AS ( - SELECT - t1.consensus_client - ,t1.execution_client - ,SUM(t1.mean * t2.distribution) AS mean - FROM - node_config_power t1 - INNER JOIN - node_distribution t2 - ON - t2.type = t1.type - GROUP BY - t1.consensus_client - ,t1.execution_client -), - -configuration_distribution AS ( - SELECT - execution_client - ,consensus_client - ,frac - FROM ( - SELECT - arrayJoin(['Erigon', 'Erigon', 'Erigon', 'Erigon', 'Nethermind', 'Nethermind', 'Nethermind', 'Nethermind']) AS execution_client, - arrayJoin(['Lighthouse', 'Teku', 'Lodestar', 'Nimbus', 'Lighthouse', 'Teku', 'Lodestar', 'Nimbus']) AS consensus_client, - arrayJoin([0.340, 0.114, 0.044, 0.002, 0.340, 0.114, 0.044, 0.002]) AS frac - ) -), - -power_best_guess AS ( - SELECT - SUM(t1.mean * t2.frac) AS mean - FROM - best_guess_per_client t1 - INNER JOIN - configuration_distribution t2 - ON - t2.execution_client = t1.execution_client - AND - t2.consensus_client = t1.consensus_client -) - - -SELECT - t1.datetime AS hour - ,t1.country_code - ,t1.cnt * t2.mean AS power -FROM - {{ ref('gnosis_nodes_geo') }} t1 -CROSS JOIN - power_best_guess t2 \ No newline at end of file diff --git a/models/metrics/attestations/consensus_atts_inclusion_distance_d.sql b/models/metrics/attestations/consensus_atts_inclusion_distance_d.sql deleted file mode 100644 index edd874c..0000000 --- a/models/metrics/attestations/consensus_atts_inclusion_distance_d.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ - config( - materialized='incremental', - incremental_strategy='insert_overwrite', - partition_by='day' - ) -}} - -WITH - - -inclusion_distance AS ( - SELECT - toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) AS day - ,inc_dist_cohort - ,SUM(cnt) AS cnt - FROM {{ ref('consensus_atts_inclusion_distance') }} - {% if is_incremental() %} - WHERE toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) >= (SELECT max(day) FROM {{ this }}) - {% endif %} - GROUP BY 1,2 -) - -SELECT * FROM inclusion_distance diff --git a/models/metrics/p2p/gnosis_p2p_nodes_forks.sqlxx b/models/metrics/p2p/gnosis_p2p_nodes_forks.sqlxx deleted file mode 100644 index fe32d67..0000000 --- a/models/metrics/p2p/gnosis_p2p_nodes_forks.sqlxx +++ /dev/null @@ -1,11 +0,0 @@ -{{ config( - materialized='incremental', - unique_key=['datetime', 'fork'], - incremental_strategy='delete+insert' -) }} - -{{ node_count_over_time( - group_by_column="next_fork_label", - source_model=ref('gnosis_p2p_nodes_status'), - grouping_column_name='fork' -) }} \ No newline at end of file diff --git a/models/metrics/p2p/gnosis_p2p_nodes_geo.sqlxx b/models/metrics/p2p/gnosis_p2p_nodes_geo.sqlxx deleted file mode 100644 index 5de26f2..0000000 --- a/models/metrics/p2p/gnosis_p2p_nodes_geo.sqlxx +++ /dev/null @@ -1,11 +0,0 @@ -{{ config( - materialized='incremental', - unique_key=['datetime', 'country'], - incremental_strategy='delete+insert' -) }} - -{{ node_count_over_time( - group_by_column="COALESCE(geo_country, 'Unknown')", - source_model=ref('gnosis_p2p_nodes_status'), - grouping_column_name='country' -) }} \ No newline at end of file diff --git a/models/metrics/p2p/gnosis_p2p_nodes_host.sqlxx b/models/metrics/p2p/gnosis_p2p_nodes_host.sqlxx deleted file mode 100644 index b35684a..0000000 --- a/models/metrics/p2p/gnosis_p2p_nodes_host.sqlxx +++ /dev/null @@ -1,11 +0,0 @@ -{{ config( - materialized='incremental', - unique_key=['datetime', 'provider'], - incremental_strategy='delete+insert' -) }} - -{{ node_count_over_time( - group_by_column="provider", - source_model=ref('gnosis_p2p_nodes_status'), - grouping_column_name='provider', -) }} \ No newline at end of file diff --git a/models/metrics/p2p/p2p_valtrack_nodes_cnt_1d.sql b/models/metrics/p2p/p2p_valtrack_nodes_cnt_1d.sql deleted file mode 100644 index d03dcb2..0000000 --- a/models/metrics/p2p/p2p_valtrack_nodes_cnt_1d.sql +++ /dev/null @@ -1 +0,0 @@ -{{ valtack_nodes_activity(resolution='day') }} \ No newline at end of file diff --git a/models/metrics/p2p/p2p_valtrack_nodes_cnt_1h.sql b/models/metrics/p2p/p2p_valtrack_nodes_cnt_1h.sql deleted file mode 100644 index 5d5f9a3..0000000 --- a/models/metrics/p2p/p2p_valtrack_nodes_cnt_1h.sql +++ /dev/null @@ -1 +0,0 @@ -{{ valtack_nodes_activity(resolution='hour') }} \ No newline at end of file diff --git a/models/metrics/p2p/p2p_valtrack_nodes_cnt_country_1d.sql b/models/metrics/p2p/p2p_valtrack_nodes_cnt_country_1d.sql deleted file mode 100644 index d7e4a7d..0000000 --- a/models/metrics/p2p/p2p_valtrack_nodes_cnt_country_1d.sql +++ /dev/null @@ -1 +0,0 @@ -{{ valtack_nodes_activity(resolution='day', aggregation_column='country') }} \ No newline at end of file diff --git a/models/metrics/p2p/p2p_valtrack_nodes_geo_last_day.sql b/models/metrics/p2p/p2p_valtrack_nodes_geo_last_day.sql deleted file mode 100644 index 05f01f3..0000000 --- a/models/metrics/p2p/p2p_valtrack_nodes_geo_last_day.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config( - materialized='table', - unique_key='enr', -) }} - -SELECT DISTINCT - t1.enr - ,t2.city - ,t2.country - ,t2.latitude - ,t2.longitude - ,t2.asn_organization - ,t2.asn_type -FROM - {{ source('valtrack','metadata_received_events') }} t1 -INNER JOIN - {{ source('valtrack','peer_discovered_events') }} t3 - ON - t3.enr = t1.enr -INNER JOIN - {{ source('valtrack','ip_metadata') }} t2 - ON t2.ip = t3.ip -WHERE - t1.timestamp >= date_trunc('day', now() - INTERVAL 14 DAY) - AND - t1.timestamp < date_trunc('day', now()- INTERVAL 13 DAY) \ No newline at end of file diff --git a/models/metrics/validators/consensus_validators_participation.sql b/models/metrics/validators/consensus_validators_participation.sql deleted file mode 100644 index 406fc52..0000000 --- a/models/metrics/validators/consensus_validators_participation.sql +++ /dev/null @@ -1,34 +0,0 @@ -{{ - config( - materialized='incremental', - incremental_strategy='insert_overwrite', - partition_by=['day'], - engine='MergeTree()', - order_by='day' - ) -}} - -WITH chunked_data AS ( - SELECT - toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) AS day, - f_aggregation_indices, - intDiv(rowNumberInAllBlocks(), 10000) AS chunk - FROM {{ get_postgres('gnosis_chaind', 't_attestations') }} - {% if is_incremental() %} - WHERE toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) >= (SELECT max(day) FROM {{ this }}) - {% endif %} -), -chunked_aggregation AS ( - SELECT - day, - chunk, - arrayDistinct(arrayFlatten(groupArrayArray(f_aggregation_indices))) AS chunk_distinct_array - FROM chunked_data - GROUP BY day, chunk -) - -SELECT - day, - length(arrayDistinct(arrayFlatten(groupArrayArray(chunk_distinct_array)))) AS distinct_count -FROM chunked_aggregation -GROUP BY day \ No newline at end of file diff --git a/models/metrics/validators/consensus_validators_participation_rate.sql b/models/metrics/validators/consensus_validators_participation_rate.sql deleted file mode 100644 index 26a2b2a..0000000 --- a/models/metrics/validators/consensus_validators_participation_rate.sql +++ /dev/null @@ -1,40 +0,0 @@ -{{ - config( - materialized='incremental', - incremental_strategy='insert_overwrite', - partition_by=['day'], - engine='MergeTree()', - order_by='day' - ) -}} - -WITH - -participation AS ( - SELECT - day, - distinct_count AS active - FROM {{ ref('consensus_validators_participation') }} -), - -validators AS ( - SELECT - day, - SUM(cnt) OVER (ORDER BY day) AS total_validators - FROM {{ ref('consensus_validators_activations') }} -) - -SELECT - p.day AS day, - p.active, - v.total_validators, - CASE - WHEN v.total_validators > 0 THEN p.active / v.total_validators - ELSE NULL - END AS participation_rate -FROM - participation p -INNER JOIN - validators v - ON p.day = v.day - diff --git a/models/metrics/validators/consensus_validators_waiting_times.sql b/models/metrics/validators/consensus_validators_waiting_times.sql deleted file mode 100644 index bf2b52b..0000000 --- a/models/metrics/validators/consensus_validators_waiting_times.sql +++ /dev/null @@ -1,30 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - -WITH -final AS ( - SELECT - toDate(activation_eligibility_time) AS day, - toInt64(activation_time) - toInt64(activation_eligibility_time) AS entry_delay, - toInt64(withdrawable_time) - toInt64(exit_time) AS exit_delay - FROM {{ ref('consensus_validators_status') }} -) - -SELECT day, COALESCE(toFloat64(min(entry_delay)),0)/3600 AS value, 'Min Entry' AS label FROM final GROUP BY 1 -UNION ALL -SELECT day, COALESCE(toFloat64(max(entry_delay)),0)/3600 AS value, 'Max Entry' AS label FROM final GROUP BY 1 -UNION ALL -SELECT day, COALESCE(toFloat64(median(entry_delay)),0)/3600 AS value, 'Median Entry' AS label FROM final GROUP BY 1 -UNION ALL -SELECT day, COALESCE(toFloat64(avg(entry_delay)),0)/3600 AS value, 'Mean Entry' AS label FROM final GROUP BY 1 -UNION ALL -SELECT day, COALESCE(toFloat64(min(exit_delay)),0)/3600 AS value, 'Min Exit' AS label FROM final GROUP BY 1 -UNION ALL -SELECT day, COALESCE(toFloat64(max(exit_delay)),0)/3600 AS value, 'Max Exit' AS label FROM final GROUP BY 1 -UNION ALL -SELECT day, COALESCE(toFloat64(median(exit_delay)),0)/3600 AS value, 'Median Exit' AS label FROM final GROUP BY 1 -UNION ALL -SELECT day, COALESCE(toFloat64(avg(exit_delay)),0)/3600 AS value, 'Mean Exit' AS label FROM final GROUP BY 1 \ No newline at end of file diff --git a/models/p2p/p2p_sources.yml b/models/p2p/p2p_sources.yml new file mode 100644 index 0000000..0df50d1 --- /dev/null +++ b/models/p2p/p2p_sources.yml @@ -0,0 +1,10 @@ +version: 2 + +sources: + - name: valtrack + schema: valtrack + tables: + - name: metadata_received_events + - name: peer_discovered_events + - name: ip_metadata + diff --git a/models/sources/p2p/p2p_sources.yml b/models/sources/p2p/p2p_sources.yml deleted file mode 100644 index 30bfaf2..0000000 --- a/models/sources/p2p/p2p_sources.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 2 - -sources: - - name: valtrack - schema: valtrack - tables: - - name: metadata_received_events - - name: peer_discovered_events - - name: ip_metadata - - - name: gnosis_chaind - database: gnosis_chaind - schema: public - tables: - - name: t_proposer_duties - - name: t_blocks - - name: t_attestations - - - name: chaind - database: chaind - schema: public - tables: - - name: t_proposer_duties - - name: t_blocks - - name: t_attestations - diff --git a/models/transformations/attestations/consensus_atts_inclusion_distance.sql b/models/transformations/attestations/consensus_atts_inclusion_distance.sql deleted file mode 100644 index b678b3b..0000000 --- a/models/transformations/attestations/consensus_atts_inclusion_distance.sql +++ /dev/null @@ -1,65 +0,0 @@ -{{ - config( - materialized='incremental', - incremental_strategy='delete+insert', - engine='ReplacingMergeTree()', - order_by='(f_inclusion_slot, inc_dist_cohort)', - primary_key='(f_inclusion_slot, inc_dist_cohort)' - ) -}} - -WITH - -attestations AS ( - SELECT - f_inclusion_slot, - f_slot, - f_inclusion_index - FROM - {{ get_postgres('gnosis_chaind', 't_attestations') }} - {% if is_incremental() %} - WHERE f_inclusion_slot > (SELECT max(f_inclusion_slot) FROM {{ this }}) - {% endif %} -), - -proposed_slots AS ( - SELECT - f_slot - FROM - {{ get_postgres('gnosis_chaind', 't_blocks') }} - {% if is_incremental() %} - WHERE f_slot > (SELECT min(f_slot) FROM attestations) - AND f_slot <= (SELECT max(f_inclusion_slot) FROM attestations) - {% endif %} -), - -slot_ranges AS ( - SELECT - a.f_inclusion_slot, - a.f_slot AS attestation_slot, - a.f_inclusion_index, - arrayJoin(range(a.f_slot + 1, a.f_inclusion_slot + 1)) AS slot - FROM attestations a -), - -inclusion_distance AS ( - SELECT - sr.f_inclusion_slot, - sr.attestation_slot, - sr.f_inclusion_index, - countDistinct(ps.f_slot) AS inc_dist_cohort - FROM slot_ranges sr - LEFT JOIN proposed_slots ps ON sr.slot = ps.f_slot - GROUP BY - sr.f_inclusion_slot, - sr.attestation_slot, - sr.f_inclusion_index -) - -SELECT - f_inclusion_slot, - inc_dist_cohort, - COUNT(*) AS cnt -FROM - inclusion_distance -GROUP BY 1, 2 \ No newline at end of file diff --git a/models/transformations/attestations/consensus_atts_inclusion_distance_old.sql b/models/transformations/attestations/consensus_atts_inclusion_distance_old.sql deleted file mode 100644 index b38eede..0000000 --- a/models/transformations/attestations/consensus_atts_inclusion_distance_old.sql +++ /dev/null @@ -1,66 +0,0 @@ -{{ - config( - materialized='incremental', - incremental_strategy='delete+insert', - engine='ReplacingMergeTree()', - order_by='(f_inclusion_slot, inc_dist_cohort)', - primary_key='(f_inclusion_slot, inc_dist_cohort)' - ) -}} - -WITH - -total_slots AS ( - SELECT - f_slot - FROM - {{ get_postgres('gnosis_chaind', 't_proposer_duties') }} - {% if is_incremental() %} - WHERE f_slot > (SELECT max(f_inclusion_slot) FROM {{ this }}) - {% endif %} -), - -proposed_slots AS ( - SELECT - f_slot - FROM - {{ get_postgres('gnosis_chaind', 't_blocks') }} - {% if is_incremental() %} - WHERE f_slot > (SELECT max(f_inclusion_slot) FROM {{ this }}) - {% endif %} -), - -attestations AS ( - SELECT - f_inclusion_slot, - f_slot, - f_inclusion_index - FROM - {{ get_postgres('gnosis_chaind', 't_attestations') }} - {% if is_incremental() %} - WHERE f_inclusion_slot > (SELECT max(f_inclusion_slot) FROM {{ this }}) - {% endif %} -), - -inclusion_distance AS ( - SELECT - a.f_inclusion_slot, - a.f_slot, - a.f_inclusion_index, - COUNT(DISTINCT p.f_slot) AS inc_dist_cohort - FROM - attestations a - CROSS JOIN - proposed_slots p - WHERE - p.f_slot > a.f_slot AND p.f_slot <= a.f_inclusion_slot - GROUP BY 1, 2, 3 -) - -SELECT - f_inclusion_slot, - inc_dist_cohort, - COUNT(*) AS cnt -FROM - inclusion_distance -GROUP BY 1, 2 diff --git a/models/transformations/p2p/gnosis_p2p_network.yml b/models/transformations/p2p/gnosis_p2p_network.yml deleted file mode 100644 index 254e9fe..0000000 --- a/models/transformations/p2p/gnosis_p2p_network.yml +++ /dev/null @@ -1,74 +0,0 @@ -version: 2 - -models: - - name: gnosis_p2p_nodes_status - meta: - blockchain: gnosis - sector: p2p - contributors: hdser - config: - tags: ['p2p', 'gnosis', 'hdser'] - description: > - Block resolution xDAI balances and balances diff for each address on gnosis. - Depends on transfers_gnosis_xdai - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - node_id - - last_seen - - last_seen_lead - - columns: - - &node_id - name: node_id - description: "Node ID" - - &last_seen - name: last_seen - description: "Last time node was seen in the network" - - &last_seen_lead - name: last_seen_lead - description: "Lead Last time node was seen in the network" - - &status - name: status - description: "Active/Inactive node" - - - name: gnosis_p2p_nodes_count - meta: - blockchain: gnosis - sector: p2p - contributors: hdser - config: - tags: ['p2p', 'gnosis', 'hdser'] - description: > - Block resolution xDAI balances and balances diff for each address on gnosis. - Depends on transfers_gnosis_xdai - columns: - - &date - name: date - description: "Last time node was seen in the network" - - &active_nodes - name: active_nodes - description: "Lead Last time node was seen in the network" - - - - name: gnosis_p2p_nodes - meta: - blockchain: gnosis - sector: p2p - contributors: hdser - config: - tags: ['p2p', 'gnosis', 'hdser'] - description: > - Block resolution xDAI balances and balances diff for each address on gnosis. - Depends on transfers_gnosis_xdai - - - name: test - meta: - blockchain: gnosis - sector: p2p - contributors: hdser - config: - tags: ['p2p', 'gnosis', 'hdser'] - description: > - Block resolution xDAI balances and balances diff for each address on gnosis. - Depends on transfers_gnosis_xdai \ No newline at end of file diff --git a/models/transformations/p2p/gnosis_p2p_nodes_status.sql b/models/transformations/p2p/gnosis_p2p_nodes_status.sql deleted file mode 100644 index 03803d0..0000000 --- a/models/transformations/p2p/gnosis_p2p_nodes_status.sql +++ /dev/null @@ -1,53 +0,0 @@ -{{ config(materialized='incremental', unique_key='last_seen') }} - -{% set activity_buffer = '72 hours' %} - -WITH - -gnosis_nodes AS ( - SELECT - enr - ,timestamp - ,IF( - timestamp = max(timestamp) OVER (PARTITION BY enr), - now(), - any(timestamp) OVER ( - PARTITION BY enr - ORDER BY timestamp ASC - ROWS BETWEEN 1 FOLLOWING AND 1 FOLLOWING - ) - ) AS timestamp_lead - FROM ( - SELECT DISTINCT - timestamp - ,enr - FROM - {{ source('valtrack','metadata_received_events') }} - ) -), - -nodes_active AS ( - SELECT - * - ,LEAST(timestamp + INTERVAL '{{ activity_buffer }}', timestamp_lead) AS active_until - FROM - gnosis_nodes -), - -nodes_status AS ( - SELECT - * - ,'active' AS status - FROM nodes_active - WHERE active_until = timestamp_lead - - UNION ALL - - SELECT - * - ,'inactive' AS status - FROM nodes_active - WHERE active_until < timestamp_lead -) - -SELECT * FROM nodes_status \ No newline at end of file diff --git a/models/transformations/p2p/test.sql b/models/transformations/p2p/test.sql deleted file mode 100644 index f64c896..0000000 --- a/models/transformations/p2p/test.sql +++ /dev/null @@ -1,14 +0,0 @@ -{{ config( - materialized='table', - engine='MergeTree()', - order_by='f_slot' -) }} - -WITH postgres_data AS ( - SELECT * - FROM {{ get_postgres('gnosis_chaind','t_blocks') }} - LIMIT 100 -) - -SELECT * -FROM postgres_data diff --git a/models/transformations/p2p/test2.sql b/models/transformations/p2p/test2.sql deleted file mode 100644 index 37dcac8..0000000 --- a/models/transformations/p2p/test2.sql +++ /dev/null @@ -1,7 +0,0 @@ - -{{ config( - materialized='table' -) }} - -SELECT * -FROM {{ flexible_source('valtrack', 'ip_metadata', 'dev') }} \ No newline at end of file diff --git a/models/transformations/power_consumption/consensus_power.sql b/models/transformations/power_consumption/consensus_power.sql deleted file mode 100644 index 4c4526e..0000000 --- a/models/transformations/power_consumption/consensus_power.sql +++ /dev/null @@ -1,14 +0,0 @@ -WITH consensus_power AS ( - SELECT - type, - client, - mean - FROM ( - SELECT - arrayJoin([4, 5, 6, 4, 5, 6, 4, 5, 6, 4, 5, 6, 4, 5, 6]) AS type, - arrayJoin(['Lighthouse', 'Lighthouse', 'Lighthouse', 'Teku', 'Teku', 'Teku', 'Lodestar', 'Lodestar', 'Lodestar', 'Nimbus', 'Nimbus', 'Nimbus', 'Prysm', 'Prysm', 'Prysm']) AS client, - arrayJoin([2.75, 3.14, 18.84, 3.71, 3.32, 27.46, 3.14, 3.89, 33.55, 1.67, 2.08, 17.11, 3.51, 2.87, 24.33]) AS mean - ) -) - -SELECT * FROM consensus_power \ No newline at end of file diff --git a/models/transformations/power_consumption/execution_power.sql b/models/transformations/power_consumption/execution_power.sql deleted file mode 100644 index 6e958d5..0000000 --- a/models/transformations/power_consumption/execution_power.sql +++ /dev/null @@ -1,15 +0,0 @@ -WITH execution_power AS ( - SELECT - type, - client, - mean - FROM ( - SELECT - arrayJoin([4, 5, 6, 4, 5, 6]) AS type, - arrayJoin(['Erigon', 'Erigon', 'Erigon', 'Nethermind', 'Nethermind', 'Nethermind']) AS client, - arrayJoin([18.6, 17.59, 44.62, 18.6, 17.59, 44.62]) AS mean - - ) -) - -SELECT * FROM execution_power \ No newline at end of file diff --git a/models/transformations/power_consumption/gnosis_nodes_geo.sql b/models/transformations/power_consumption/gnosis_nodes_geo.sql deleted file mode 100644 index 690035e..0000000 --- a/models/transformations/power_consumption/gnosis_nodes_geo.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config( - materialized='incremental', - unique_key=['datetime', 'country_code'], - incremental_strategy='delete+insert' -) }} - -{{ node_count_over_time( - group_by_column="COALESCE(geo_country_code, 'Unknown')", - source_model=ref('gnosis_p2p_nodes_status'), - grouping_column_name='country_code' -) }} \ No newline at end of file diff --git a/models/transformations/power_consumption/hardware_config.sql b/models/transformations/power_consumption/hardware_config.sql deleted file mode 100644 index 94acac0..0000000 --- a/models/transformations/power_consumption/hardware_config.sql +++ /dev/null @@ -1,28 +0,0 @@ -WITH hardware_config AS ( - SELECT - type - ,cpu - ,cores_threads - ,architecture - ,ram - ,storage - ,gpu - ,psu - ,"case" - ,os - FROM ( - SELECT - arrayJoin([4, 5, 6]) AS type, - arrayJoin(['Intel i5-1135G7', 'Intel i5-10400', 'AMD 3970X']) AS cpu, - arrayJoin(['4/8', '6/12', '32/64']) AS cores_threads, - arrayJoin(['x86/x64', 'x86/x64', 'x86/x64']) AS architecture, - arrayJoin(['16 GB', '64 GB', '256 GB']) AS ram, - arrayJoin(['2 TB SSD', '2TB SSD', '2TB SSD']) AS storage, - arrayJoin(['Onboard', 'Onboard', 'AM 6970']) AS gpu, - arrayJoin(['65 Watt', '650 Watt', '1000 Watt']) AS psu, - arrayJoin(['Integrated', 'Custom', 'Custom']) AS "case", - arrayJoin(['Ubuntu 20.04', 'Ubuntu 21', 'Ubuntu 20.04']) AS os - ) -) - -SELECT * FROM hardware_config \ No newline at end of file diff --git a/models/transformations/power_consumption/idle_electric_power.sql b/models/transformations/power_consumption/idle_electric_power.sql deleted file mode 100644 index be776a9..0000000 --- a/models/transformations/power_consumption/idle_electric_power.sql +++ /dev/null @@ -1,12 +0,0 @@ -WITH idle_electric_power AS ( - SELECT - type - ,mean - FROM ( - SELECT - arrayJoin([4, 5, 6]) AS type, - arrayJoin([3.66, 25.04, 78.17]) AS mean - ) -) - -SELECT * FROM idle_electric_power \ No newline at end of file diff --git a/models/transformations/power_consumption/node_distribution.sql b/models/transformations/power_consumption/node_distribution.sql deleted file mode 100644 index 06fb254..0000000 --- a/models/transformations/power_consumption/node_distribution.sql +++ /dev/null @@ -1,12 +0,0 @@ -WITH node_distribution AS ( - SELECT - type - ,distribution - FROM ( - SELECT - arrayJoin([4, 5, 6]) AS type, - arrayJoin([0.25, 0.50, 0.25]) AS distribution - ) -) - -SELECT * FROM node_distribution \ No newline at end of file diff --git a/models/transformations/validators/consensus_validators_status.sql b/models/transformations/validators/consensus_validators_status.sql deleted file mode 100644 index 57a1851..0000000 --- a/models/transformations/validators/consensus_validators_status.sql +++ /dev/null @@ -1,25 +0,0 @@ -{{ - config( - materialized='incremental', - incremental_strategy='delete+insert' - ) -}} - -WITH - -validators AS ( - SELECT - f_index - ,f_withdrawal_credentials - ,{{ compute_timestamp_at_epoch('f_activation_eligibility_epoch') }} AS activation_eligibility_time - ,{{ compute_timestamp_at_epoch('f_activation_epoch') }} AS activation_time - ,{{ compute_timestamp_at_epoch('f_exit_epoch') }} AS exit_time - ,{{ compute_timestamp_at_epoch('f_withdrawable_epoch') }} AS withdrawable_time - FROM - {{ get_postgres('gnosis_chaind', 't_validators') }} - {% if is_incremental() %} - WHERE f_inclusion_slot > (SELECT max(f_inclusion_slot) FROM {{ this }}) - {% endif %} -) - -SELECT * FROM validators \ No newline at end of file diff --git a/models/transformations/validators/test_chaind.sql b/models/transformations/validators/test_chaind.sql deleted file mode 100644 index ff5c10e..0000000 --- a/models/transformations/validators/test_chaind.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ - config( - materialized='table' - ) -}} - -WITH - -validators AS ( - SELECT - COUNT(*) - FROM - {{ get_postgres('chaind', 't_attestations') }} -) - -SELECT * FROM validators \ No newline at end of file diff --git a/profiles.yml b/profiles.yml index 963b5e5..cdee81b 100644 --- a/profiles.yml +++ b/profiles.yml @@ -14,16 +14,6 @@ gnosis_dbt: connect_timeout: 60 read_timeout: 3000 - gnosis_chaind: - type: postgres - dbname: "gnosis_chaind" - schema: "public" - host: "{{ env_var('POSTGRES_HOST', 'postgres') }}" - user: "{{ env_var('POSTGRES_USER') }}" - password: "{{ env_var('POSTGRES_PASSWORD') }}" - port: "{{ env_var('POSTGRES_PORT', '5432') | int }}" - threads: 80 - chaind: type: postgres dbname: "chaind" diff --git a/scripts/ymls/requirements.txt b/scripts/ymls/requirements.txt new file mode 100644 index 0000000..fbc97b7 --- /dev/null +++ b/scripts/ymls/requirements.txt @@ -0,0 +1,2 @@ +pyyaml==6.0.1 +sqlparse==0.5.1 \ No newline at end of file diff --git a/scripts/ymls/ymls_generator.py b/scripts/ymls/ymls_generator.py new file mode 100644 index 0000000..0355b51 --- /dev/null +++ b/scripts/ymls/ymls_generator.py @@ -0,0 +1,162 @@ +import os +import yaml +import re +from pathlib import Path +import sqlparse + +def extract_columns_from_sql(sql_file_path): + """Extract column names and attempt to infer descriptions from SQL file.""" + with open(sql_file_path, 'r') as f: + sql_content = f.read() + + # Parse SQL content + parsed = sqlparse.parse(sql_content)[0] + + # Look for SELECT statements and CTEs + columns = set() + for token in parsed.tokens: + if isinstance(token, sqlparse.sql.Token): + # Extract column names from SELECT statements + if token.ttype is sqlparse.tokens.DML and token.value.upper() == 'SELECT': + select_columns = re.findall(r'SELECT\s+(.+?)\s+FROM', sql_content, re.IGNORECASE | re.DOTALL) + if select_columns: + cols = select_columns[0].split(',') + for col in cols: + # Extract column name, handling aliases + col = col.strip() + if ' AS ' in col.upper(): + col = col.split(' AS ')[-1] + # Remove any table qualifiers + col = col.split('.')[-1] + # Clean up any remaining whitespace or quotes + col = col.strip('` "\'\n\t') + if col and not col.startswith('('): + columns.add(col) + + # Create column entries + column_entries = [] + for col in sorted(columns): + entry = { + 'name': col, + 'description': f"TODO: Add description for {col}", + 'data_tests': [ + 'not_null', + 'unique' + ] + } + column_entries.append(entry) + + return column_entries + +def get_metadata_from_path(path): + """Extract metadata from the path structure.""" + parts = path.parts + + # Get the first part after 'models' as the blockchain + blockchain_idx = parts.index('models') + 1 + blockchain = parts[blockchain_idx] if blockchain_idx < len(parts) else None + + # Get the next part as the sector + sector_idx = blockchain_idx + 1 + sector = parts[sector_idx] if sector_idx < len(parts) else None + + # Build tags from the path components + tags = [] + if blockchain: + tags.append(blockchain) + if sector: + tags.append(sector) + + # Add additional tags based on folder structure + for part in parts[sector_idx+1:]: + if part not in ['metrics', 'transformations'] and not part.endswith('.sql'): + tags.append(part) + + return { + 'blockchain': blockchain, + 'sector': sector, + 'tags': tags + } + +def create_model_entry(file_path): + """Create a complete model entry including metadata, config, and columns.""" + model_name = os.path.splitext(os.path.basename(file_path))[0] + metadata = get_metadata_from_path(Path(file_path)) + + # Extract columns from SQL file + columns = extract_columns_from_sql(file_path) + + # Get first two column names for unique combination test + first_two_columns = [col['name'] for col in columns[:2]] + + model_entry = { + 'name': model_name, + 'meta': { + 'blockchain': metadata['blockchain'], + 'sector': metadata['sector'], + 'contributors': 'TODO: Add contributors' + }, + 'config': { + 'tags': metadata['tags'] + }, + 'description': f'TODO: Add description for {model_name}', + 'columns': columns + } + + # Add tests if we have at least two columns + if len(first_two_columns) >= 2: + model_entry['data_tests'] = [ + { + 'dbt_utils.unique_combination_of_columns': { + 'combination_of_columns': first_two_columns + } + } + ] + + return model_entry + +def create_schema_yaml(directory_path, models): + """Create a schema.yml file with the given models.""" + schema_content = { + 'version': 2, + 'models': models + } + + schema_path = os.path.join(directory_path, 'schema.yml') + with open(schema_path, 'w') as f: + yaml.dump(schema_content, f, sort_keys=False, default_flow_style=False, allow_unicode=True) + +def process_directory(models_path): + """Process directory recursively and create schema.yml files.""" + for root, dirs, files in os.walk(models_path): + # Skip if no SQL files in directory + sql_files = [f for f in files if f.endswith('.sql')] + if not sql_files: + continue + + # Create model entries for each SQL file + models = [] + for sql_file in sql_files: + file_path = os.path.join(root, sql_file) + model_entry = create_model_entry(file_path) + models.append(model_entry) + + # Create schema.yml if we have models + if models: + create_schema_yaml(root, models) + print(f"Created schema.yml in {root}") + +def main(): + # Find the models directory + current_dir = os.getcwd() + models_path = os.path.join(current_dir, 'models') + + if not os.path.exists(models_path): + print(f"Error: {models_path} directory not found") + return + + process_directory(models_path) + print("Schema files generation completed!") + +if __name__ == "__main__": + main() \ No newline at end of file From 02a41ce33327aaf33d30cb4e5a42af915cc789ce Mon Sep 17 00:00:00 2001 From: hdser Date: Sat, 2 Nov 2024 13:39:56 +0100 Subject: [PATCH 4/4] models updated --- cron.sh | 2 +- cron_preview.sh | 2 +- macros/db/get_incremental_filter.sql | 14 +++ .../consensus_atts_inclusion_distance.sql | 35 +++++++ .../consensus_atts_inclusion_distance_d.sql | 28 ------ .../consensus/attestations/metrics/schema.yml | 9 +- ...ql => consensus_blocks_graffiti_top10.sql} | 2 +- .../metrics/consensus_blocks_production.sql | 74 +++++++++----- models/consensus/blocks/metrics/schema.yml | 4 +- .../consensus_blocks_graffiti.sql | 37 ++++--- .../blocks/transformations/schema.yml | 5 - .../consensus_validators_activations.sql | 81 ++++++++------- .../consensus_validators_entry_queue.sql | 99 +++++++++---------- ...onsensus_validators_participation_rate.sql | 36 ++++--- .../consensus_validators_queue_cnt_d.sql | 56 ----------- .../consensus_validators_waiting_times.sql | 13 +-- .../consensus/validators/metrics/schema.yml | 68 ++----------- .../consensus_validators_queue.sql | 78 ++++++++------- 18 files changed, 300 insertions(+), 343 deletions(-) create mode 100644 macros/db/get_incremental_filter.sql create mode 100644 models/consensus/attestations/metrics/consensus_atts_inclusion_distance.sql delete mode 100644 models/consensus/attestations/metrics/consensus_atts_inclusion_distance_d.sql rename models/consensus/blocks/metrics/{consensus_blocks_graffiti_top10_d.sql => consensus_blocks_graffiti_top10.sql} (97%) delete mode 100644 models/consensus/validators/metrics/consensus_validators_queue_cnt_d.sql diff --git a/cron.sh b/cron.sh index 6316653..5a206d7 100755 --- a/cron.sh +++ b/cron.sh @@ -1,6 +1,6 @@ #!/bin/bash -dbt run -s execution_power +dbt run diff --git a/cron_preview.sh b/cron_preview.sh index 6316653..5a206d7 100755 --- a/cron_preview.sh +++ b/cron_preview.sh @@ -1,6 +1,6 @@ #!/bin/bash -dbt run -s execution_power +dbt run diff --git a/macros/db/get_incremental_filter.sql b/macros/db/get_incremental_filter.sql new file mode 100644 index 0000000..e05dbad --- /dev/null +++ b/macros/db/get_incremental_filter.sql @@ -0,0 +1,14 @@ +{% macro get_incremental_filter() %} + {% if is_incremental() %} + last_partition AS ( + SELECT max(partition_month) as partition_month + FROM {{ this }} + ), + {% endif %} +{% endmacro %} + +{% macro apply_incremental_filter(timestamp_field, add_and=false) %} + {% if is_incremental() %} + {{ "AND " if add_and else "WHERE "}}toStartOfMonth({{ timestamp_field }}) >= (SELECT partition_month FROM last_partition) + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/models/consensus/attestations/metrics/consensus_atts_inclusion_distance.sql b/models/consensus/attestations/metrics/consensus_atts_inclusion_distance.sql new file mode 100644 index 0000000..97f371b --- /dev/null +++ b/models/consensus/attestations/metrics/consensus_atts_inclusion_distance.sql @@ -0,0 +1,35 @@ +{{ + config( + materialized='incremental', + incremental_strategy='delete+insert', + engine='ReplacingMergeTree()', + order_by='(day, inc_dist_cohort)', + unique_key='(day, inc_dist_cohort)', + partition_by='partition_month' + ) +}} + +WITH + +{{ get_incremental_filter() }} + + +inclusion_distance AS ( + SELECT + toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) AS day + ,f_inclusion_slot - f_slot AS inc_dist_cohort + ,COUNT(*) AS cnt + FROM {{ get_postgres('chaind', 't_attestations') }} + {{ apply_incremental_filter(compute_timestamp_at_slot('f_inclusion_slot')) }} + GROUP BY 1,2 +) + +SELECT + toStartOfMonth(day) AS partition_month + ,day + ,inc_dist_cohort + ,cnt +FROM + inclusion_distance +WHERE + day < (SELECT MAX(day) FROM inclusion_distance) diff --git a/models/consensus/attestations/metrics/consensus_atts_inclusion_distance_d.sql b/models/consensus/attestations/metrics/consensus_atts_inclusion_distance_d.sql deleted file mode 100644 index 4b5cd75..0000000 --- a/models/consensus/attestations/metrics/consensus_atts_inclusion_distance_d.sql +++ /dev/null @@ -1,28 +0,0 @@ -{{ - config( - materialized='incremental', - incremental_strategy='delete+insert', - engine='ReplacingMergeTree()', - order_by='(day, inc_dist_cohort)', - primary_key='(day, inc_dist_cohort)', - partition_by='partition_month' - ) -}} - -WITH - - -inclusion_distance AS ( - SELECT - toStartOfMonth({{ compute_timestamp_at_slot('f_inclusion_slot') }}) AS partition_month - ,toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) AS day - ,f_inclusion_slot - f_slot AS inc_dist_cohort - ,COUNT(*) AS cnt - FROM {{ get_postgres('chaind', 't_attestations') }} - {% if is_incremental() %} - WHERE toStartOfMonth({{ compute_timestamp_at_slot('f_inclusion_slot') }}) >= (SELECT max(partition_month) FROM {{ this }}) - {% endif %} - GROUP BY 1,2,3 -) - -SELECT * FROM inclusion_distance diff --git a/models/consensus/attestations/metrics/schema.yml b/models/consensus/attestations/metrics/schema.yml index 5d8030b..3fc139e 100644 --- a/models/consensus/attestations/metrics/schema.yml +++ b/models/consensus/attestations/metrics/schema.yml @@ -1,6 +1,6 @@ version: 2 models: -- name: consensus_atts_inclusion_distance_d +- name: consensus_atts_inclusion_distance meta: blockchain: consensus sector: attestations @@ -9,7 +9,7 @@ models: tags: - consensus - attestations - description: 'TODO: Add description for consensus_atts_inclusion_distance_d' + description: 'TODO: Add description for consensus_atts_inclusion_distance' columns: - name: cnt description: 'TODO: Add description for cnt' @@ -26,11 +26,6 @@ models: data_tests: - not_null - unique - - name: partition_month - description: 'TODO: Add description for partition_month' - data_tests: - - not_null - - unique data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: diff --git a/models/consensus/blocks/metrics/consensus_blocks_graffiti_top10_d.sql b/models/consensus/blocks/metrics/consensus_blocks_graffiti_top10.sql similarity index 97% rename from models/consensus/blocks/metrics/consensus_blocks_graffiti_top10_d.sql rename to models/consensus/blocks/metrics/consensus_blocks_graffiti_top10.sql index 6470a90..f6a2f08 100644 --- a/models/consensus/blocks/metrics/consensus_blocks_graffiti_top10_d.sql +++ b/models/consensus/blocks/metrics/consensus_blocks_graffiti_top10.sql @@ -4,7 +4,7 @@ incremental_strategy='delete+insert', engine='ReplacingMergeTree()', order_by='(day, graffiti_top10)', - primary_key='(day, graffiti_top10)', + unique_key='(day, graffiti_top10)', partition_by='partition_month', settings={ "allow_nullable_key": 1 diff --git a/models/consensus/blocks/metrics/consensus_blocks_production.sql b/models/consensus/blocks/metrics/consensus_blocks_production.sql index ee04044..6134258 100644 --- a/models/consensus/blocks/metrics/consensus_blocks_production.sql +++ b/models/consensus/blocks/metrics/consensus_blocks_production.sql @@ -1,16 +1,25 @@ {{ config( materialized='incremental', - incremental_strategy='insert_overwrite', - partition_by='toStartOfWeek(day)' + incremental_strategy='delete+insert', + engine='ReplacingMergeTree()', + order_by='(day, label)', + unique_key='(day, label)', + partition_by='partition_month' ) }} -WITH genesis AS ( +WITH + +{{ get_incremental_filter() }} + + +genesis AS ( SELECT f_time AS genesis_time FROM {{ get_postgres('chaind', 't_genesis') }} LIMIT 1 ), + blocks AS ( SELECT toDate({{ compute_timestamp_at_slot('f_slot') }}) AS day, @@ -19,40 +28,51 @@ blocks AS ( CAST(SUM(IF(f_canonical, 1, 0)) AS Int64) AS proposed FROM {{ get_postgres('chaind', 't_blocks') }} - {% if is_incremental() %} - WHERE toStartOfWeek(toDate({{ compute_timestamp_at_slot('f_slot') }})) >= (SELECT max(toStartOfWeek(day)) FROM {{ this }}) - {% endif %} + {{ apply_incremental_filter(compute_timestamp_at_slot('f_slot')) }} GROUP BY 1 ), + chain_specs AS ( SELECT toInt64OrZero({{ get_chain_spec('SECONDS_PER_SLOT') }}) AS seconds_per_slot, {{ seconds_until_end_of_day('genesis.genesis_time') }} AS seconds_until_end_of_day FROM genesis -) +), -SELECT day, forked AS cnt, 'forked' AS label -FROM blocks +final AS ( + SELECT day, forked AS cnt, 'forked' AS label + FROM blocks -UNION ALL + UNION ALL -SELECT day, proposed AS cnt, 'proposed' AS label -FROM blocks + SELECT day, proposed AS cnt, 'proposed' AS label + FROM blocks -UNION ALL + UNION ALL + + SELECT + blocks.day, + CASE + WHEN blocks.f_slot_start = 0 + THEN CAST( + (chain_specs.seconds_until_end_of_day) / chain_specs.seconds_per_slot - (blocks.proposed + blocks.forked) AS Int64 + ) + ELSE + CAST( + (24 * 3600) / chain_specs.seconds_per_slot - (blocks.proposed + blocks.forked) AS Int64 + ) + END AS cnt, + 'missed' AS label + FROM blocks + CROSS JOIN chain_specs +) SELECT - blocks.day, - CASE - WHEN blocks.f_slot_start = 0 - THEN CAST( - (chain_specs.seconds_until_end_of_day) / chain_specs.seconds_per_slot - (blocks.proposed + blocks.forked) AS Int64 - ) - ELSE - CAST( - (24 * 3600) / chain_specs.seconds_per_slot - (blocks.proposed + blocks.forked) AS Int64 - ) - END AS cnt, - 'missed' AS label -FROM blocks -CROSS JOIN chain_specs \ No newline at end of file + toStartOfMonth(day) AS partition_month + ,day + ,cnt + ,label +FROM + final +WHERE + day < (SELECT MAX(day) FROM final) \ No newline at end of file diff --git a/models/consensus/blocks/metrics/schema.yml b/models/consensus/blocks/metrics/schema.yml index 5d5a783..7cc5fe2 100644 --- a/models/consensus/blocks/metrics/schema.yml +++ b/models/consensus/blocks/metrics/schema.yml @@ -26,7 +26,7 @@ models: combination_of_columns: - proposer_index - withdrawable_time -- name: consensus_blocks_graffiti_top10_d +- name: consensus_blocks_graffiti_top10 meta: blockchain: consensus sector: blocks @@ -35,7 +35,7 @@ models: tags: - consensus - blocks - description: 'TODO: Add description for consensus_blocks_graffiti_top10_d' + description: 'TODO: Add description for consensus_blocks_graffiti_top10' columns: - name: cnt description: 'TODO: Add description for cnt' diff --git a/models/consensus/blocks/transformations/consensus_blocks_graffiti.sql b/models/consensus/blocks/transformations/consensus_blocks_graffiti.sql index 99d22fb..8630d9f 100644 --- a/models/consensus/blocks/transformations/consensus_blocks_graffiti.sql +++ b/models/consensus/blocks/transformations/consensus_blocks_graffiti.sql @@ -1,28 +1,37 @@ {{ - config( - materialized='incremental', - incremental_strategy='delete+insert', + config( + materialized='incremental', + incremental_strategy='delete+insert', engine='ReplacingMergeTree()', order_by='(day, graffiti, f_proposer_index)', - primary_key='(day, graffiti, f_proposer_index)', - partition_by='partition_month' - ) + unique_key='(day, graffiti, f_proposer_index)', + partition_by='partition_month' + ) }} +WITH -WITH blocks_graffiti AS ( +{{ get_incremental_filter() }} + +blocks_graffiti AS ( SELECT - toStartOfMonth({{ compute_timestamp_at_slot('f_slot') }}) AS partition_month - ,toDate({{ compute_timestamp_at_slot('f_slot') }}) AS day + toDate({{ compute_timestamp_at_slot('f_slot') }}) AS day ,f_proposer_index ,{{ decode_graffiti('f_graffiti') }} AS graffiti ,COUNT(*) AS cnt FROM {{ get_postgres('chaind', 't_blocks') }} - {% if is_incremental() %} - WHERE toStartOfMonth({{ compute_timestamp_at_slot('f_slot') }}) >= (SELECT max(partition_month) FROM {{ this }}) - {% endif %} - GROUP BY 1, 2, 3, 4 + {{ apply_incremental_filter(compute_timestamp_at_slot('f_slot')) }} + GROUP BY 1, 2, 3 ) -SELECT * FROM blocks_graffiti +SELECT + toStartOfMonth(day) AS partition_month + ,day + ,f_proposer_index + ,graffiti + ,cnt +FROM + blocks_graffiti +WHERE + day < (SELECT MAX(day) FROM blocks_graffiti) diff --git a/models/consensus/blocks/transformations/schema.yml b/models/consensus/blocks/transformations/schema.yml index 41eaa85..dfb4f48 100644 --- a/models/consensus/blocks/transformations/schema.yml +++ b/models/consensus/blocks/transformations/schema.yml @@ -31,11 +31,6 @@ models: data_tests: - not_null - unique - - name: partition_month - description: 'TODO: Add description for partition_month' - data_tests: - - not_null - - unique data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: diff --git a/models/consensus/validators/metrics/consensus_validators_activations.sql b/models/consensus/validators/metrics/consensus_validators_activations.sql index d6d256c..018f242 100644 --- a/models/consensus/validators/metrics/consensus_validators_activations.sql +++ b/models/consensus/validators/metrics/consensus_validators_activations.sql @@ -1,53 +1,66 @@ {{ config( materialized='incremental', - incremental_strategy='insert_overwrite', - partition_by='toYYYYMM(coalesce(month, toDate(\'1970-01-01\')))' + incremental_strategy='delete+insert', + partition_by='partition_month', + order_by='(day, label)', + unique_key='(day, label)', + settings={ + "allow_nullable_key": 1 + } ) }} -WITH validators_activations AS ( +WITH + +{{ get_incremental_filter() }} + +validators_activations AS ( SELECT - toDate(activation_time) AS day, - toStartOfMonth(toDate(activation_time)) AS month, - CAST(COUNT(*) AS Int64) AS cnt, - 'activations' AS label + toDate(activation_time) AS day + ,CAST(COUNT(*) AS Int64) AS cnt + ,'activations' AS label FROM {{ ref('consensus_validators_queue') }} - {% if is_incremental() %} - WHERE toDate(activation_time) >= (SELECT max(day) FROM {{ this }}) - {% endif %} - GROUP BY 1, 2 + {{ apply_incremental_filter('f_eth1_block_timestamp') }} + GROUP BY 1, 3 ), validators_exits AS ( SELECT - toDate(exit_time) AS day, - toStartOfMonth(toDate(exit_time)) AS month, - -CAST(COUNT(*) AS Int64) AS cnt, - 'exits' AS label + toDate(exit_time) AS day + ,-CAST(COUNT(*) AS Int64) AS cnt + ,'exits' AS label FROM {{ ref('consensus_validators_queue') }} WHERE exit_time IS NOT NULL - {% if is_incremental() %} - AND toDate(exit_time) >= (SELECT max(day) FROM {{ this }}) - {% endif %} - GROUP BY 1, 2 -) + {{ apply_incremental_filter('f_eth1_block_timestamp', add_and=true) }} + GROUP BY 1, 3 +), -SELECT - day, - coalesce(month, toDate('1970-01-01')) AS month, - cnt, - label -FROM validators_activations +final AS ( + SELECT + day, + cnt, + label + FROM validators_activations + WHERE day IS NOT NULL -UNION ALL + UNION ALL -SELECT - day, - coalesce(month, toDate('1970-01-01')) AS month, - cnt, - label -FROM validators_exits + SELECT + day, + cnt, + label + FROM validators_exits + WHERE day IS NOT NULL +) -ORDER BY day, label \ No newline at end of file +SELECT + toStartOfMonth(day) AS partition_month + ,day + ,cnt + ,label +FROM + final +WHERE + day < (SELECT MAX(day) FROM final) diff --git a/models/consensus/validators/metrics/consensus_validators_entry_queue.sql b/models/consensus/validators/metrics/consensus_validators_entry_queue.sql index b5c23b7..a890c19 100644 --- a/models/consensus/validators/metrics/consensus_validators_entry_queue.sql +++ b/models/consensus/validators/metrics/consensus_validators_entry_queue.sql @@ -1,8 +1,10 @@ {{ config( materialized='incremental', - incremental_strategy='insert_overwrite', + incremental_strategy='delete+insert', partition_by='partition_month', + order_by='(day)', + unique_key='(day)', settings={ "allow_nullable_key": 1 } @@ -11,76 +13,67 @@ WITH -{% if is_incremental() %} -last_partition AS ( - SELECT max(partition_month) as partition_month - FROM {{ this }} -), -{% endif %} +{{ get_incremental_filter() }} validators_in_activation_queue AS ( SELECT - toDate(f_eth1_block_timestamp) AS day, - toStartOfMonth(toDate(f_eth1_block_timestamp)) AS partition_month, - CAST(COUNT(*) AS Int64) AS cnt + toDate(f_eth1_block_timestamp) AS day + ,CAST(COUNT(*) AS Int64) AS cnt FROM {{ ref('consensus_validators_queue') }} - {% if is_incremental() %} - WHERE toStartOfMonth(toDate(f_eth1_block_timestamp)) >= (SELECT partition_month FROM last_partition) - {% endif %} - GROUP BY 1, 2 + {{ apply_incremental_filter('f_eth1_block_timestamp') }} + GROUP BY 1 ), validators_activated AS ( SELECT - toDate(activation_time) AS day, - toStartOfMonth(toDate(activation_time)) AS partition_month, - CAST(COUNT(*) AS Int64) AS cnt + toDate(activation_time) AS day + ,CAST(COUNT(*) AS Int64) AS cnt FROM {{ ref('consensus_validators_queue') }} - {% if is_incremental() %} - WHERE toStartOfMonth(toDate(f_eth1_block_timestamp)) >= (SELECT partition_month FROM last_partition) - {% endif %} - GROUP BY 1, 2 + {{ apply_incremental_filter('f_eth1_block_timestamp') }} + GROUP BY 1 ), -min_max_dates AS ( - SELECT - min(dates.day) as min_date, - max(dates.day) as max_date +all_dates AS ( + SELECT arrayJoin( + arrayMap( + d -> toDate(min_date + d), + range(0, dateDiff('day', min_date, max_date) + 1) + ) + ) AS day FROM ( - SELECT day FROM validators_in_activation_queue - UNION ALL - SELECT day FROM validators_activated - ) dates -), - -calendar AS ( - SELECT - toDate(min_date + number) AS day, - toStartOfMonth(toDate(min_date + number)) AS partition_month - FROM min_max_dates - CROSS JOIN ( - SELECT arrayJoin(range(dateDiff('day', min_date, max_date) + 1)) AS number - FROM min_max_dates - ) s + SELECT + min(dates.day) as min_date, + max(dates.day) as max_date + FROM ( + SELECT day FROM validators_in_activation_queue + UNION ALL + SELECT day FROM validators_activated + ) dates + WHERE dates.day IS NOT NULL + ) + WHERE min_date IS NOT NULL + AND max_date IS NOT NULL ), daily_metrics AS ( SELECT - c.day AS day, - c.partition_month As partition_month, - COALESCE(q.cnt, 0) AS validators_entered_queue, - COALESCE(a.cnt, 0) AS validators_activated, - COALESCE(q.cnt, 0) - COALESCE(a.cnt, 0) AS net_queue_change - FROM calendar c + c.day AS day + ,COALESCE(q.cnt, 0) AS validators_entered_queue + ,COALESCE(a.cnt, 0) AS validators_activated + ,COALESCE(q.cnt, 0) - COALESCE(a.cnt, 0) AS net_queue_change + FROM all_dates c LEFT JOIN validators_in_activation_queue q ON c.day = q.day LEFT JOIN validators_activated a ON c.day = a.day + WHERE c.day IS NOT NULL ) - SELECT - day, - partition_month, - validators_entered_queue, - validators_activated, - net_queue_change -FROM daily_metrics \ No newline at end of file + toStartOfMonth(day) AS partition_month + ,day + ,validators_entered_queue + ,validators_activated + ,net_queue_change +FROM + daily_metrics +WHERE + day < (SELECT MAX(day) FROM daily_metrics) \ No newline at end of file diff --git a/models/consensus/validators/metrics/consensus_validators_participation_rate.sql b/models/consensus/validators/metrics/consensus_validators_participation_rate.sql index 4ae1131..be0e22f 100644 --- a/models/consensus/validators/metrics/consensus_validators_participation_rate.sql +++ b/models/consensus/validators/metrics/consensus_validators_participation_rate.sql @@ -4,28 +4,40 @@ incremental_strategy='delete+insert', engine='ReplacingMergeTree()', order_by='(day)', - primary_key='(day)', - partition_by='partition_month' + unique_key='(day)', + partition_by='partition_month' ) }} -WITH committe_size AS ( +WITH + +{{ get_incremental_filter() }} + +committe_size AS ( SELECT CAST(f_value AS FLOAT) AS f_value FROM {{ get_postgres('chaind', 't_chain_spec') }} WHERE f_key = 'EPOCHS_PER_SYNC_COMMITTEE_PERIOD' +), + +final AS ( + SELECT + toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) AS day + ,AVG(length(f_indices)/(SELECT f_value FROM committe_size)) AS mean_pct + FROM + {{ get_postgres('chaind', 't_sync_aggregates') }} + {{ apply_incremental_filter(compute_timestamp_at_slot('f_inclusion_slot')) }} + GROUP BY 1 ) SELECT - toStartOfMonth(toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }})) AS partition_month - ,toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }}) AS day - ,AVG(length(f_indices)/(SELECT f_value FROM committe_size)) AS mean_pct -FROM - {{ get_postgres('chaind', 't_sync_aggregates') }} -{% if is_incremental() %} -WHERE toStartOfMonth(toDate({{ compute_timestamp_at_slot('f_inclusion_slot') }})) >= (SELECT max(partition_month) FROM {{ this }}) -{% endif %} -GROUP BY 1, 2 + toStartOfMonth(day) AS partition_month + ,day + ,mean_pct +FROM + final +WHERE + day < (SELECT MAX(day) FROM final) diff --git a/models/consensus/validators/metrics/consensus_validators_queue_cnt_d.sql b/models/consensus/validators/metrics/consensus_validators_queue_cnt_d.sql deleted file mode 100644 index fb9647a..0000000 --- a/models/consensus/validators/metrics/consensus_validators_queue_cnt_d.sql +++ /dev/null @@ -1,56 +0,0 @@ -{{ - config( - materialized='table', - ) -}} - -{% set delay_fields = [ - {'field': 'eligibility_delay', 'label': 'Eligibility'}, - {'field': 'activation_delay', 'label': 'Activation'}, - {'field': 'entry_delay', 'label': 'Entry'}, - {'field': 'exit_activation_delay', 'label': 'De-activation'}, - {'field': 'exit_withdrawable_delay', 'label': 'Withdrawable'}, - {'field': 'queue_exit_delay', 'label': 'Exit'}, - {'field': 'exit_delay', 'label': 'Exit Delay'} -] %} - -{% set aggregations = [ - {'func': 'min', 'label': 'Min'}, - {'func': 'max', 'label': 'Max'}, - {'func': 'median', 'label': 'Median'}, - {'func': 'avg', 'label': 'Mean'} -] %} - -WITH final AS ( - SELECT - f_validator_pubkey, - MIN(f_eth1_block_timestamp) AS eth1_block_timestamp, - MAX(toInt64(activation_eligibility_time) - toInt64(f_eth1_block_timestamp)) AS eligibility_delay, - MAX(toInt64(activation_time) - toInt64(activation_eligibility_time)) AS activation_delay, - MAX(toInt64(activation_time) - toInt64(f_eth1_block_timestamp)) AS entry_delay, - MAX(toInt64(exit_time) - toInt64(exit_request_time)) AS exit_activation_delay, - MAX(toInt64(withdrawable_time) - toInt64(exit_time)) AS exit_withdrawable_delay, - MAX(toInt64(withdrawable_time) - toInt64(exit_request_time)) AS queue_exit_delay, - MAX(toInt64(withdrawable_time) - toInt64(exit_voluntary_time)) AS exit_delay - FROM - {{ ref('consensus_validators_queue') }} - GROUP BY - f_validator_pubkey -) - -{% set union_queries = [] %} -{% for delay in delay_fields %} - {% for agg in aggregations %} - {% set query %} -SELECT - toDate(eth1_block_timestamp) AS day, - COALESCE(toFloat64({{ agg.func }}({{ delay.field }})), 0) / 3600 AS value, - '{{ agg.label }} {{ delay.label }}' AS label -FROM final -GROUP BY day - {% endset %} - {% do union_queries.append(query) %} - {% endfor %} -{% endfor %} - -{{ union_queries | join('\nUNION ALL\n') }} \ No newline at end of file diff --git a/models/consensus/validators/metrics/consensus_validators_waiting_times.sql b/models/consensus/validators/metrics/consensus_validators_waiting_times.sql index fb9647a..d12c8ca 100644 --- a/models/consensus/validators/metrics/consensus_validators_waiting_times.sql +++ b/models/consensus/validators/metrics/consensus_validators_waiting_times.sql @@ -42,12 +42,13 @@ WITH final AS ( {% for delay in delay_fields %} {% for agg in aggregations %} {% set query %} -SELECT - toDate(eth1_block_timestamp) AS day, - COALESCE(toFloat64({{ agg.func }}({{ delay.field }})), 0) / 3600 AS value, - '{{ agg.label }} {{ delay.label }}' AS label -FROM final -GROUP BY day + SELECT + toDate(eth1_block_timestamp) AS day, + COALESCE(toFloat64({{ agg.func }}({{ delay.field }})), 0) / 3600 AS value, + '{{ agg.label }} {{ delay.label }}' AS label + FROM final + WHERE toDate(eth1_block_timestamp) < (SELECT MAX(toDate(eth1_block_timestamp)) FROM final) + GROUP BY day {% endset %} {% do union_queries.append(query) %} {% endfor %} diff --git a/models/consensus/validators/metrics/schema.yml b/models/consensus/validators/metrics/schema.yml index 869350b..cef464d 100644 --- a/models/consensus/validators/metrics/schema.yml +++ b/models/consensus/validators/metrics/schema.yml @@ -134,11 +134,6 @@ models: data_tests: - not_null - unique - - name: month - description: 'TODO: Add description for month' - data_tests: - - not_null - - unique data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: @@ -155,69 +150,18 @@ models: - validators description: 'TODO: Add description for consensus_validators_entry_queue' columns: - - name: max(partition_month) as partition_month - description: 'TODO: Add description for max(partition_month) as partition_month' - data_tests: - - not_null - - unique -- name: consensus_validators_queue_cnt_d - meta: - blockchain: consensus - sector: validators - contributors: 'TODO: Add contributors' - config: - tags: - - consensus - - validators - description: 'TODO: Add description for consensus_validators_queue_cnt_d' - columns: - - name: activation_delay - description: 'TODO: Add description for activation_delay' - data_tests: - - not_null - - unique - - name: eligibility_delay - description: 'TODO: Add description for eligibility_delay' - data_tests: - - not_null - - unique - - name: entry_delay - description: 'TODO: Add description for entry_delay' - data_tests: - - not_null - - unique - - name: eth1_block_timestamp - description: 'TODO: Add description for eth1_block_timestamp' - data_tests: - - not_null - - unique - - name: exit_activation_delay - description: 'TODO: Add description for exit_activation_delay' - data_tests: - - not_null - - unique - - name: exit_delay - description: 'TODO: Add description for exit_delay' - data_tests: - - not_null - - unique - - name: exit_withdrawable_delay - description: 'TODO: Add description for exit_withdrawable_delay' - data_tests: - - not_null - - unique - - name: f_validator_pubkey - description: 'TODO: Add description for f_validator_pubkey' + - name: cnt + description: 'TODO: Add description for cnt' data_tests: - not_null - unique - - name: queue_exit_delay - description: 'TODO: Add description for queue_exit_delay' + - name: day + description: 'TODO: Add description for day' data_tests: - not_null - unique data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - activation_delay - - eligibility_delay + - cnt + - day diff --git a/models/consensus/validators/transformations/consensus_validators_queue.sql b/models/consensus/validators/transformations/consensus_validators_queue.sql index 15180fb..2031abd 100644 --- a/models/consensus/validators/transformations/consensus_validators_queue.sql +++ b/models/consensus/validators/transformations/consensus_validators_queue.sql @@ -4,42 +4,52 @@ incremental_strategy='delete+insert', engine='ReplacingMergeTree()', order_by='(f_eth1_block_timestamp, f_validator_pubkey)', - primary_key='(f_eth1_block_timestamp, f_validator_pubkey)', + unique_key='(f_eth1_block_timestamp, f_validator_pubkey)', partition_by='partition_month' ) }} -SELECT - toStartOfMonth(el.f_eth1_block_timestamp) AS partition_month - ,v.f_index - ,el.f_validator_pubkey AS f_validator_pubkey - ,el.f_withdrawal_credentials - ,el.f_signature - ,el.f_eth1_block_timestamp AS f_eth1_block_timestamp - ,el.f_eth1_gas_used - ,el.f_eth1_gas_price - ,el.f_amount - ,{{ compute_timestamp_at_slot('cl.f_inclusion_slot') }} AS inclusion_time - ,{{ compute_timestamp_at_epoch('v.f_activation_eligibility_epoch') }} AS activation_eligibility_time - ,{{ compute_timestamp_at_epoch('v.f_activation_epoch') }} AS activation_time - ,{{ compute_timestamp_at_slot('ve.f_inclusion_slot') }} AS exit_request_time - ,{{ compute_timestamp_at_epoch('ve.f_epoch') }} AS exit_voluntary_time - ,{{ compute_timestamp_at_epoch('v.f_exit_epoch') }} AS exit_time - ,{{ compute_timestamp_at_epoch('v.f_withdrawable_epoch') }} AS withdrawable_time +WITH + +{{ get_incremental_filter() }} + +final AS ( + SELECT + toStartOfMonth(el.f_eth1_block_timestamp) AS partition_month + ,v.f_index + ,el.f_validator_pubkey AS f_validator_pubkey + ,el.f_withdrawal_credentials + ,el.f_signature + ,el.f_eth1_block_timestamp AS f_eth1_block_timestamp + ,el.f_eth1_gas_used + ,el.f_eth1_gas_price + ,el.f_amount + ,{{ compute_timestamp_at_slot('cl.f_inclusion_slot') }} AS inclusion_time + ,{{ compute_timestamp_at_epoch('v.f_activation_eligibility_epoch') }} AS activation_eligibility_time + ,{{ compute_timestamp_at_epoch('v.f_activation_epoch') }} AS activation_time + ,{{ compute_timestamp_at_slot('ve.f_inclusion_slot') }} AS exit_request_time + ,{{ compute_timestamp_at_epoch('ve.f_epoch') }} AS exit_voluntary_time + ,{{ compute_timestamp_at_epoch('v.f_exit_epoch') }} AS exit_time + ,{{ compute_timestamp_at_epoch('v.f_withdrawable_epoch') }} AS withdrawable_time + FROM + {{ get_postgres('chaind', 't_eth1_deposits') }} el + LEFT JOIN + {{ get_postgres('chaind', 't_deposits') }} cl + ON cl.f_validator_pubkey = el.f_validator_pubkey + INNER JOIN + {{ get_postgres('chaind', 't_validators') }} v + ON v.f_public_key = el.f_validator_pubkey + LEFT JOIN + {{ get_postgres('chaind', 't_voluntary_exits') }} ve + ON ve.f_validator_index = v.f_index + {{ apply_incremental_filter('el.f_eth1_block_timestamp') }} + SETTINGS + join_use_nulls=1 +) + +SELECT + * FROM - {{ get_postgres('chaind', 't_eth1_deposits') }} el -LEFT JOIN - {{ get_postgres('chaind', 't_deposits') }} cl - ON cl.f_validator_pubkey = el.f_validator_pubkey -INNER JOIN - {{ get_postgres('chaind', 't_validators') }} v - ON v.f_public_key = el.f_validator_pubkey -LEFT JOIN - {{ get_postgres('chaind', 't_voluntary_exits') }} ve - ON ve.f_validator_index = v.f_index -{% if is_incremental() %} -WHERE - toStartOfWeek(el.f_eth1_block_timestamp) >= (SELECT max(partition_week) FROM {{ this }}) -{% endif %} -SETTINGS - join_use_nulls = 1 \ No newline at end of file + final + +