From db4ae17f9a416b42f9d01aef519edd9e2c4abaec Mon Sep 17 00:00:00 2001 From: Ivan De Marino Date: Sun, 15 Oct 2023 23:15:17 +0100 Subject: [PATCH 1/6] Desp upgrade --- Cargo.lock | 49 ++++++++++++++++++++++++------------------------- Cargo.toml | 6 +++--- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d602464..d508864 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,9 +91,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", @@ -281,18 +281,18 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "const_format" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c990efc7a285731f9a4378d81aff2f0e85a2c8781a05ef0f8baa8dac54d0ff48" +checksum = "e3a214c7af3d04997541b18d432afaff4c455e79e2029079647e72fc2bd27673" dependencies = [ "const_format_proc_macros", ] [[package]] name = "const_format_proc_macros" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e026b6ce194a874cb9cf32cd5772d1ef9767cc8fcb5765948d74f37a9d8b2bf6" +checksum = "c7f6ff08fd20f4f299298a28e2dfa8a8ba1036e6cd2460ac1de7b425d76f2500" dependencies = [ "proc-macro2", "quote", @@ -982,9 +982,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.6" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" +checksum = "aaac441002f822bc9705a681810a4dd2963094b9ca0ddc41cb963a4c189189ea" dependencies = [ "aho-corasick", "memchr", @@ -994,9 +994,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.9" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +checksum = "5011c7e263a695dc8ca064cddb722af1be54e517a280b12a5356f98366899e5d" dependencies = [ "aho-corasick", "memchr", @@ -1005,9 +1005,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "rustc-demangle" @@ -1017,9 +1017,9 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.18" +version = "0.38.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" +checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ "bitflags 2.4.0", "errno", @@ -1061,18 +1061,18 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" dependencies = [ "proc-macro2", "quote", @@ -1319,11 +1319,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-core", @@ -1331,9 +1330,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] @@ -1545,9 +1544,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907" +checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index 87d1d03..af4d53c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,11 +27,11 @@ exclude = [ # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -async-trait = "0.1.73" +async-trait = "0.1.74" axum = { version = "0.6.20", features = ["http2"] } chrono = "0.4.31" clap = { version = "4.4.6", features = ["derive", "deprecated", "env", "wrap_help"] } -const_format = "0.2.31" +const_format = "0.2.32" ctrlc = { version = "3.4.1", features = ["termination"] } env_logger = "0.10.0" exit-code = "1.0.0" @@ -39,7 +39,7 @@ hyper = { version = "0.14.27", features = ["http1", "http2", "server", "runtime" konsumer_offsets = { version = "0.3.0", default-features = false, features = ["ts_chrono"] } log = "0.4.20" prometheus = "0.13.3" -regex = "1.9.6" +regex = "1.10.1" thiserror = "1.0.49" tokio = { version = "1.33.0", features = ["rt", "rt-multi-thread", "time", "sync", "macros"] } tokio-util = "0.7.9" From 10165d633467dea707c8206244ff5040066ff17d Mon Sep 17 00:00:00 2001 From: Ivan De Marino Date: Sat, 4 Nov 2023 22:20:55 +0000 Subject: [PATCH 2/6] Adding `consumer_groups` emitter Cluster and Kommitted (internal) metrics --- src/consumer_groups/emitter.rs | 80 ++++++++++++++++++++++++++++++++-- src/consumer_groups/mod.rs | 6 ++- src/main.rs | 16 ++++--- src/prometheus_metrics/mod.rs | 1 + 4 files changed, 92 insertions(+), 11 deletions(-) diff --git a/src/consumer_groups/emitter.rs b/src/consumer_groups/emitter.rs index 5da1652..93ed7fa 100644 --- a/src/consumer_groups/emitter.rs +++ b/src/consumer_groups/emitter.rs @@ -1,7 +1,12 @@ -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; use async_trait::async_trait; +use const_format::formatcp; use konsumer_offsets::ConsumerProtocolAssignment; +use prometheus::{Histogram, HistogramOpts, IntGauge, IntGaugeVec, Opts, Registry}; use rdkafka::{admin::AdminClient, client::DefaultClientContext, groups::GroupList, ClientConfig}; use tokio::{ sync::mpsc, @@ -13,12 +18,25 @@ use tokio_util::sync::CancellationToken; use crate::constants::KOMMITTED_CONSUMER_OFFSETS_CONSUMER; use crate::internals::Emitter; use crate::kafka_types::{Group, GroupWithMembers, Member, MemberWithAssignment, TopicPartition}; +use crate::prometheus_metrics::{LABEL_GROUP, NAMESPACE}; const CHANNEL_SIZE: usize = 5; const FETCH_TIMEOUT: Duration = Duration::from_secs(10); const FETCH_INTERVAL: Duration = Duration::from_secs(60); +const M_CG_NAME: &str = formatcp!("{NAMESPACE}_consumer_groups_total"); +const M_CG_HELP: &str = "Consumer groups currently in the cluster"; +const M_CG_MEMBERS_NAME: &str = formatcp!("{NAMESPACE}_consumer_groups_members_total"); +const M_CG_MEMBERS_HELP: &str = "Members of consumer groups currently in the cluster"; +const M_CG_FETCH_NAME: &str = + formatcp!("{NAMESPACE}_consumer_groups_emitter_fetch_time_milliseconds"); +const M_CG_FETCH_HELP: &str = + "Time (in milliseconds) taken to fetch information about all consumer groups in cluster"; +const M_CG_CH_CAP_NAME: &str = formatcp!("{NAMESPACE}_consumer_groups_emitter_channel_capacity"); +const M_CG_CH_CAP_HELP: &str = + "Capacity of internal channel used to send consumer groups metadata to rest of the service"; + /// A map of all the known Consumer Groups, at a given point in time. /// /// This reflects the internal state of Kafka and it's active Consumer Groups. @@ -95,6 +113,12 @@ impl From for ConsumerGroups { /// It shuts down when the provided [`CancellationToken`] is cancelled. pub struct ConsumerGroupsEmitter { admin_client_config: ClientConfig, + + // Prometheus Metrics + metric_cg: IntGauge, + metric_cg_members: IntGaugeVec, + metric_cg_fetch: Histogram, + metric_cg_ch_cap: IntGauge, } impl ConsumerGroupsEmitter { @@ -103,9 +127,39 @@ impl ConsumerGroupsEmitter { /// # Arguments /// /// * `admin_client_config` - Kafka admin client configuration, used to fetch Consumer Groups - pub fn new(admin_client_config: ClientConfig) -> Self { + pub fn new(admin_client_config: ClientConfig, metrics: Arc) -> Self { + // Create metrics + let metric_cg = IntGauge::new(M_CG_NAME, M_CG_HELP) + .unwrap_or_else(|_| panic!("Failed to create metric: {M_CG_NAME}")); + let metric_cg_members = + IntGaugeVec::new(Opts::new(M_CG_MEMBERS_NAME, M_CG_MEMBERS_HELP), &[LABEL_GROUP]) + .unwrap_or_else(|_| panic!("Failed to create metric: {M_CG_MEMBERS_NAME}")); + let metric_cg_fetch = + Histogram::with_opts(HistogramOpts::new(M_CG_FETCH_NAME, M_CG_FETCH_HELP)) + .unwrap_or_else(|_| panic!("Failed to create metric: {M_CG_FETCH_NAME}")); + let metric_cg_ch_cap = IntGauge::new(M_CG_CH_CAP_NAME, M_CG_CH_CAP_HELP) + .unwrap_or_else(|_| panic!("Failed to create metric: {M_CG_CH_CAP_NAME}")); + + // Register metrics + metrics + .register(Box::new(metric_cg.clone())) + .unwrap_or_else(|_| panic!("Failed to register metric: {M_CG_NAME}")); + metrics + .register(Box::new(metric_cg_members.clone())) + .unwrap_or_else(|_| panic!("Failed to register metric: {M_CG_MEMBERS_NAME}")); + metrics + .register(Box::new(metric_cg_fetch.clone())) + .unwrap_or_else(|_| panic!("Failed to register metric: {M_CG_FETCH_NAME}")); + metrics + .register(Box::new(metric_cg_ch_cap.clone())) + .unwrap_or_else(|_| panic!("Failed to register metric: {M_CG_CH_CAP_NAME}")); + Self { admin_client_config, + metric_cg, + metric_cg_members, + metric_cg_fetch, + metric_cg_ch_cap, } } } @@ -133,19 +187,37 @@ impl Emitter for ConsumerGroupsEmitter { let (sx, rx) = mpsc::channel::(CHANNEL_SIZE); + // Clone metrics so they can be used in the spawned future + let metric_cg = self.metric_cg.clone(); + let metric_cg_members = self.metric_cg_members.clone(); + let metric_cg_fetch = self.metric_cg_fetch.clone(); + let metric_cg_ch_cap = self.metric_cg_ch_cap.clone(); + let join_handle = tokio::spawn(async move { let mut interval = interval(FETCH_INTERVAL); loop { + let timer = metric_cg_fetch.start_timer(); let res_groups = admin_client .inner() .fetch_group_list(None, FETCH_TIMEOUT) .map(Self::Emitted::from); + // Update fetching time metric + timer.observe_duration(); + match res_groups { - Ok(groups) => { + Ok(cg) => { + // Update group and group member metrics + metric_cg.set(cg.groups.len() as i64); + for (g, gm) in cg.groups.iter() { + metric_cg_members.with_label_values(&[&g]).set(gm.members.len() as i64); + } + // Update channel capacity metric + metric_cg_ch_cap.set(sx.capacity() as i64); + tokio::select! { - res = Self::emit_with_interval(&sx, groups, &mut interval) => { + res = Self::emit_with_interval(&sx, cg, &mut interval) => { if let Err(e) = res { error!("Failed to emit {}: {e}", std::any::type_name::()); } diff --git a/src/consumer_groups/mod.rs b/src/consumer_groups/mod.rs index 1ed898d..b73b692 100644 --- a/src/consumer_groups/mod.rs +++ b/src/consumer_groups/mod.rs @@ -1,6 +1,9 @@ // Inner module mod emitter; +use std::sync::Arc; + +use prometheus::Registry; use rdkafka::ClientConfig; use tokio::sync::mpsc::Receiver; use tokio::task::JoinHandle; @@ -13,8 +16,9 @@ pub use emitter::{ConsumerGroups, ConsumerGroupsEmitter}; pub fn init( admin_client_config: ClientConfig, shutdown_token: CancellationToken, + metrics: Arc, ) -> (Receiver, JoinHandle<()>) { - let consumer_groups_emitter = ConsumerGroupsEmitter::new(admin_client_config); + let consumer_groups_emitter = ConsumerGroupsEmitter::new(admin_client_config, metrics); let (cg_rx, cg_join) = consumer_groups_emitter.spawn(shutdown_token); debug!("Initialized"); diff --git a/src/main.rs b/src/main.rs index c883057..844afbf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -37,6 +37,10 @@ async fn main() -> Result<(), Box> { cs_reg.await_ready(shutdown_token.clone()).await?; let cs_reg_arc = Arc::new(cs_reg); + // Init `prometheus_metrics` module + let prom_reg = prometheus_metrics::init(cs_reg_arc.get_cluster_id().await); + let prom_reg_arc = Arc::new(prom_reg); + // Init `partition_offsets` module, and await registry to be ready let (po_reg, po_join) = partition_offsets::init( admin_client_config.clone(), @@ -53,17 +57,17 @@ async fn main() -> Result<(), Box> { konsumer_offsets_data::init(admin_client_config.clone(), shutdown_token.clone()); // Init `consumer_groups` module - let (cg_rx, cg_join) = - consumer_groups::init(admin_client_config.clone(), shutdown_token.clone()); + let (cg_rx, cg_join) = consumer_groups::init( + admin_client_config.clone(), + shutdown_token.clone(), + prom_reg_arc.clone(), + ); // Init `lag_register` module, and await registry to be ready let lag_reg = lag_register::init(cg_rx, kod_rx, po_reg_arc.clone()); lag_reg.await_ready(shutdown_token.clone()).await?; let lag_reg_arc = Arc::new(lag_reg); - // Init `prometheus_metrics` module - let prom_reg = prometheus_metrics::init(cs_reg_arc.get_cluster_id().await); - // Init `http` module let http_fut = http::init( cli.listen_on(), @@ -71,7 +75,7 @@ async fn main() -> Result<(), Box> { po_reg_arc.clone(), lag_reg_arc.clone(), shutdown_token.clone(), - Arc::new(prom_reg), + prom_reg_arc.clone(), ); // Join all the async tasks, then let it terminate diff --git a/src/prometheus_metrics/mod.rs b/src/prometheus_metrics/mod.rs index 1d9d7d5..b02f595 100644 --- a/src/prometheus_metrics/mod.rs +++ b/src/prometheus_metrics/mod.rs @@ -18,6 +18,7 @@ pub const UNKNOWN_VAL: &str = "UNKNOWN"; pub fn init(cluster_id: String) -> Registry { let prom_def_labels = HashMap::from([(LABEL_CLUSTER_ID.to_string(), cluster_id)]); + Registry::new_custom(Some(NAMESPACE.to_string()), Some(prom_def_labels)) .expect("Unable to create a Prometheus Metrics Registry") } From 95a1d8f8ca53871efa31c6efd6499a33f722cf86 Mon Sep 17 00:00:00 2001 From: Ivan De Marino Date: Sat, 4 Nov 2023 22:21:14 +0000 Subject: [PATCH 3/6] Removing references to issues addressed by this work --- src/http/mod.rs | 2 -- src/internals/emitter.rs | 1 - 2 files changed, 3 deletions(-) diff --git a/src/http/mod.rs b/src/http/mod.rs index 1bb1661..acd1039 100644 --- a/src/http/mod.rs +++ b/src/http/mod.rs @@ -209,12 +209,10 @@ async fn prometheus_metrics(State(state): State) -> impl IntoR // // --- CLUSTER METRICS --- // - // TODO https://github.com/kafkesc/kommitted/issues/53 // TODO https://github.com/kafkesc/kommitted/issues/54 // // --- KOMMITTED INTERNAL METRICS --- // - // TODO https://github.com/kafkesc/kommitted/issues/55 // TODO https://github.com/kafkesc/kommitted/issues/56 // TODO https://github.com/kafkesc/kommitted/issues/57 diff --git a/src/internals/emitter.rs b/src/internals/emitter.rs index e52ad8e..e3ad7d8 100644 --- a/src/internals/emitter.rs +++ b/src/internals/emitter.rs @@ -56,7 +56,6 @@ pub trait Emitter { // TODO Each `Emitter` implementation should report a metric about // the current saturation of its emitting channel. - // See https://github.com/kafkesc/kommitted/issues/55 // See https://github.com/kafkesc/kommitted/issues/56 // See https://github.com/kafkesc/kommitted/issues/57 From 3315b41daa79c20924851f9e729d13a9b220fb44 Mon Sep 17 00:00:00 2001 From: Ivan De Marino Date: Sat, 4 Nov 2023 22:21:25 +0000 Subject: [PATCH 4/6] Documented new metrics in `METRICS.md` --- METRICS.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/METRICS.md b/METRICS.md index 96babdb..678a3c7 100644 --- a/METRICS.md +++ b/METRICS.md @@ -87,6 +87,54 @@ Below is the list of the current Metrics exposed by Kommitted. +### Cluster Metrics + +Those are metrics specific to the component in Kommitted that fetches the set of Consumer Groups and Members from +the Kafka cluster, and sends it to the rest of the system for further processing (i.e. to know which consumer groups +to produce the lag information for). + +
+
kmtd_consumer_groups_total
+
+ Description: "Consumer groups currently in the cluster.
+ Labels: cluster_id
+ Type: gauge
+ Timestamped: true +
+
+ +
+
kmtd_consumer_groups_members_total
+
+ Description: Members of consumer groups currently in the cluster.
+ Labels: cluster_id, group
+ Type: gauge
+ Timestamped: true +
+
+ +### Kommitted (internal) Metrics + +
+
kmtd_consumer_groups_emitter_fetch_time_milliseconds
+
+ Description: Time (in milliseconds) taken to fetch information about all consumer groups in cluster.
+ Labels: cluster_id
+ Type: histogram
+ Timestamped: true +
+
+ +
+
kmtd_consumer_groups_emitter_channel_capacity
+
+ Description: Capacity of internal channel used to send consumer groups metadata to rest of the service.
+ Labels: cluster_id
+ Type: gauge
+ Timestamped: true +
+
+ ## Labels Each metrics has some or all of the following labels applied; what labels applies From 590dbb2effdf8ae18ff869a24d2c1c903908cc84 Mon Sep 17 00:00:00 2001 From: Ivan De Marino Date: Sat, 4 Nov 2023 22:23:37 +0000 Subject: [PATCH 5/6] Deps upgrade --- Cargo.lock | 176 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 8 +-- 2 files changed, 92 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d508864..6651187 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -178,9 +178,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "bumpalo" @@ -234,9 +234,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.6" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" dependencies = [ "clap_builder", "clap_derive", @@ -244,9 +244,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.6" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" dependencies = [ "anstream", "anstyle", @@ -257,9 +257,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", @@ -269,9 +269,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "colorchoice" @@ -379,36 +379,36 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-core", "futures-task", @@ -449,9 +449,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" [[package]] name = "heck" @@ -522,7 +522,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -531,16 +531,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -564,12 +564,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown 0.14.2", ] [[package]] @@ -591,9 +591,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" dependencies = [ "wasm-bindgen", ] @@ -673,9 +673,9 @@ checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -716,9 +716,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "wasi", @@ -731,7 +731,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "cfg-if", "libc", ] @@ -793,18 +793,18 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "openssl-src" -version = "300.1.5+3.1.3" +version = "300.1.6+3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "559068e4c12950d7dcaa1857a61725c0d38d4fc03ff8e070ab31a75d6e316491" +checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" dependencies = [ "cc", "libc", @@ -835,9 +835,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", @@ -973,18 +973,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.10.1" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaac441002f822bc9705a681810a4dd2963094b9ca0ddc41cb963a4c189189ea" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", @@ -994,9 +994,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5011c7e263a695dc8ca064cddb722af1be54e517a280b12a5356f98366899e5d" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", @@ -1017,11 +1017,11 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.19" +version = "0.38.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", @@ -1061,18 +1061,18 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.189" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" +checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.189" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" +checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" dependencies = [ "proc-macro2", "quote", @@ -1081,9 +1081,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -1139,9 +1139,9 @@ checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -1149,9 +1149,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", "windows-sys", @@ -1212,18 +1212,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", @@ -1242,7 +1242,7 @@ dependencies = [ "mio", "num_cpus", "pin-project-lite", - "socket2 0.5.4", + "socket2 0.5.5", "tokio-macros", "windows-sys", ] @@ -1260,9 +1260,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -1274,9 +1274,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" [[package]] name = "toml_edit" @@ -1284,7 +1284,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.1.0", "toml_datetime", "winnow", ] @@ -1319,9 +1319,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.39" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "log", "pin-project-lite", @@ -1384,9 +1384,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1394,9 +1394,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" dependencies = [ "bumpalo", "log", @@ -1409,9 +1409,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1419,9 +1419,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ "proc-macro2", "quote", @@ -1432,9 +1432,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" [[package]] name = "winapi" @@ -1468,10 +1468,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ "windows-targets", ] @@ -1544,9 +1544,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.17" +version = "0.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c" +checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index af4d53c..1db2adf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ exclude = [ async-trait = "0.1.74" axum = { version = "0.6.20", features = ["http2"] } chrono = "0.4.31" -clap = { version = "4.4.6", features = ["derive", "deprecated", "env", "wrap_help"] } +clap = { version = "4.4.7", features = ["derive", "deprecated", "env", "wrap_help"] } const_format = "0.2.32" ctrlc = { version = "3.4.1", features = ["termination"] } env_logger = "0.10.0" @@ -39,10 +39,10 @@ hyper = { version = "0.14.27", features = ["http1", "http2", "server", "runtime" konsumer_offsets = { version = "0.3.0", default-features = false, features = ["ts_chrono"] } log = "0.4.20" prometheus = "0.13.3" -regex = "1.10.1" -thiserror = "1.0.49" +regex = "1.10.2" +thiserror = "1.0.50" tokio = { version = "1.33.0", features = ["rt", "rt-multi-thread", "time", "sync", "macros"] } -tokio-util = "0.7.9" +tokio-util = "0.7.10" [target.'cfg(unix)'.dependencies] rdkafka = { version = "0.34.0", features = ["ssl-vendored", "gssapi-vendored", "libz-static"] } From e9aff574bf219d073793c124c3b99ecf240b8126 Mon Sep 17 00:00:00 2001 From: Ivan De Marino Date: Sat, 4 Nov 2023 22:39:55 +0000 Subject: [PATCH 6/6] Remove empty lines from Prometheus `/metrics` output --- src/http/mod.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/http/mod.rs b/src/http/mod.rs index acd1039..065659b 100644 --- a/src/http/mod.rs +++ b/src/http/mod.rs @@ -100,7 +100,6 @@ async fn prometheus_metrics(State(state): State) -> impl IntoR consumer_partition_offset::append_headers(&mut body); iter_lag_reg(&state.lag_reg, &mut body, &cluster_id, consumer_partition_offset::append_metric) .await; - body.push(String::new()); // ------------------------------------------------------- METRIC: consumer_partition_lag_offset consumer_partition_lag_offset::append_headers(&mut body); @@ -111,7 +110,6 @@ async fn prometheus_metrics(State(state): State) -> impl IntoR consumer_partition_lag_offset::append_metric, ) .await; - body.push(String::new()); // ------------------------------------------------- METRIC: consumer_partition_lag_milliseconds consumer_partition_lag_milliseconds::append_headers(&mut body); @@ -122,7 +120,6 @@ async fn prometheus_metrics(State(state): State) -> impl IntoR consumer_partition_lag_milliseconds::append_metric, ) .await; - body.push(String::new()); // ------------------------------------------------- METRIC: partition_earliest_available_offset partition_earliest_available_offset::append_headers(&mut body); @@ -142,7 +139,6 @@ async fn prometheus_metrics(State(state): State) -> impl IntoR }, } } - body.push(String::new()); // ------------------------------------------------- METRIC: partition_latest_available_offset partition_latest_available_offset::append_headers(&mut body); @@ -162,7 +158,6 @@ async fn prometheus_metrics(State(state): State) -> impl IntoR }, } } - body.push(String::new()); // ------------------------------------------------- METRIC: partition_earliest_tracked_offset partition_earliest_tracked_offset::append_headers(&mut body); @@ -183,7 +178,6 @@ async fn prometheus_metrics(State(state): State) -> impl IntoR }, } } - body.push(String::new()); // ------------------------------------------------- METRIC: partition_latest_tracked_offset partition_latest_tracked_offset::append_headers(&mut body); @@ -204,13 +198,11 @@ async fn prometheus_metrics(State(state): State) -> impl IntoR }, } } - body.push(String::new()); - // // --- CLUSTER METRICS --- // // TODO https://github.com/kafkesc/kommitted/issues/54 - // + // --- KOMMITTED INTERNAL METRICS --- // // TODO https://github.com/kafkesc/kommitted/issues/56 @@ -219,7 +211,7 @@ async fn prometheus_metrics(State(state): State) -> impl IntoR // Turn the bespoke metrics created so far, into a String let mut body = body.join("\n"); - // Append the the bespoke metrics, internal (normal?) Prometheus Metrics + // Append to the bespoke metrics, classic Prometheus Metrics let metrics_family = state.metrics.gather(); if let Err(e) = TextEncoder.encode_utf8(&metrics_family, &mut body) { status = StatusCode::INTERNAL_SERVER_ERROR;