Merge pull request #25 from rodneylab/test__update_metrics_testing

test: ☑️ improve metrics testing

Cargo.toml

@@ -49,6 +49,7 @@ tracing-opentelemetry = "0.27.0"
 tracing-subscriber = { version = "0.3.18", features = ["std", "env-filter"] }
 
 [dev-dependencies]
+float-cmp = "0.10.0"
 futures = "0.3.31"
 http-body-util = "0.1.2"
 insta = { version = "1.41.1", features = ["glob", "json", "redactions"] }

src/observability/metrics.rs

@@ -23,7 +23,7 @@ pub(crate) fn create_prometheus_recorder() -> PrometheusHandle {
             panic!("Could not initialise the bucket for '{REQUEST_DURATION_METRIC_NAME}'",)
         })
         .install_recorder()
-        .expect("Could not install the Prometheus recorder")
+        .expect("Could not install the Prometheus recorder, there might already be an instance running.  It should only be started once.")
 }
 
 pub(crate) async fn track_metrics(req: Request, next: Next) -> impl IntoResponse {
@@ -54,16 +54,18 @@ pub(crate) async fn track_metrics(req: Request, next: Next) -> impl IntoResponse
 
 #[cfg(test)]
 mod tests {
-    use std::str;
+    use std::{collections::BTreeMap, str};
 
     use axum::{
         body::Body,
         http::{Request, StatusCode},
         Router,
     };
+    use float_cmp::approx_eq;
     use http_body_util::BodyExt;
     use metrics_exporter_prometheus::PrometheusHandle;
     use once_cell::sync::Lazy;
+    use prometheus_parse::Scrape;
     use sqlx::sqlite::SqlitePoolOptions;
     use tower::ServiceExt;
 
@@ -107,8 +109,33 @@ mod tests {
 
         // assert
         assert_eq!(response.status(), StatusCode::OK);
-        let body = response.into_body().collect().await.unwrap().to_bytes();
-        assert_eq!(&body[..], b"");
+    }
+
+    /// Internally, `prometheus_parse` uses an [`std::collections::HashMap`] to store labels, which means the order of the
+    /// values will not be deterministic and so not suitable for snapshot testing.  This helper
+    /// function creates a [`BTreeMap`] from the labels, and will return labels in a deterministic
+    /// order.
+    fn sorted_prometheus_metric_labels(labels: &prometheus_parse::Labels) -> BTreeMap<&str, &str> {
+        labels
+            .iter()
+            .fold(BTreeMap::<&str, &str>::new(), |mut acc, (key, val)| {
+                acc.insert(key, val);
+                acc
+            })
+    }
+
+    #[tokio::test]
+    #[should_panic(
+        expected = "Could not install the Prometheus recorder, there might already be an instance running.  It should only be started once.: FailedToSetGlobalRecorder(SetRecorderError { .. })"
+    )]
+    async fn create_prometheus_emits_error_message_if_called_more_than_once() {
+        // arrange
+        let _ = Lazy::force(&METRICS);
+
+        // act
+        create_prometheus_recorder();
+
+        // assert
     }
 
     #[tokio::test]
@@ -117,15 +144,14 @@ mod tests {
         // Avoid re-initialising the tracing subscriber for each test
         let recorder_handle = Lazy::force(&METRICS);
         Lazy::force(&TRACING);
-        //let app = get_app().await;
         Lazy::force(&TRACING);
         std::env::set_var("OPENTELEMETRY_ENABLED", "true");
         let main_app_instance = get_app().await;
         let metrics_app_instance = metrics_app(recorder_handle.clone());
 
         // act
         let _ = main_app_instance
-            .oneshot(Request::get("/health_check").body(Body::empty()).unwrap())
+            .oneshot(Request::get("/health").body(Body::empty()).unwrap())
             .await
             .unwrap();
         let response = metrics_app_instance
@@ -137,10 +163,66 @@ mod tests {
         assert_eq!(response.status(), StatusCode::OK);
         let body_bytes = response.into_body().collect().await.unwrap().to_bytes();
         let body = str::from_utf8(&body_bytes).unwrap();
-        let mut lines = body.lines();
-        assert_eq!(lines.next(), Some("# TYPE http_requests_total counter"));
-
-        let line_2 = lines.next().unwrap();
-        assert_eq!(line_2.find("http_requests_total"), Some(0));
+        let lines: Vec<_> = body.lines().map(|val| Ok(val.to_owned())).collect();
+        let Scrape { docs, samples } = prometheus_parse::Scrape::parse(lines.into_iter()).unwrap();
+        assert!(docs.is_empty());
+        assert!(samples.len() > 3);
+        assert_eq!(samples.len() % 4, 0);
+
+        let metric = "http_requests_duration_seconds";
+        let sample = samples
+            .iter()
+            .find(|val| val.metric == metric && val.labels.get("path") == Some("/health"))
+            .unwrap_or_else(|| panic!("Missing `{metric}` metric"));
+        let prometheus_parse::Value::Histogram(histogram) = &sample.value else {
+            panic!("Expected histogram, got {:?}", sample.value);
+        };
+        assert_eq!(histogram.len(), 12);
+        assert!(histogram[0].count >= 1.0);
+        assert!(approx_eq!(
+            f64,
+            histogram[0].less_than,
+            0.005,
+            epsilon = f64::EPSILON,
+            ulps = 2
+        ));
+        let labels = sorted_prometheus_metric_labels(&sample.labels);
+        insta::assert_json_snapshot!(labels);
+
+        let metric = "http_requests_duration_seconds_count";
+        let sample = samples
+            .iter()
+            .find(|val| val.metric == metric && val.labels.get("path") == Some("/health"))
+            .unwrap_or_else(|| panic!("Missing `{metric}` metric"));
+        let prometheus_parse::Value::Untyped(count) = &sample.value else {
+            panic!("Expected time count, got {:?}", sample.value);
+        };
+        assert!(*count <= 10.0);
+        let labels = sorted_prometheus_metric_labels(&sample.labels);
+        insta::assert_json_snapshot!(labels);
+
+        let metric = "http_requests_duration_seconds_sum";
+        let sample = samples
+            .iter()
+            .find(|val| val.metric == metric && val.labels.get("path") == Some("/health"))
+            .unwrap_or_else(|| panic!("Missing `{metric}` metric"));
+        let prometheus_parse::Value::Untyped(sum) = &sample.value else {
+            panic!("Expected time sum, got {:?}", sample.value);
+        };
+        assert!(*sum <= 0.001);
+        let labels = sorted_prometheus_metric_labels(&sample.labels);
+        insta::assert_json_snapshot!(labels);
+
+        let metric = "http_requests_total";
+        let sample = samples
+            .iter()
+            .find(|val| val.metric == metric && val.labels.get("path") == Some("/health"))
+            .unwrap_or_else(|| panic!("Missing `{metric}` metric"));
+        let prometheus_parse::Value::Counter(total) = &sample.value else {
+            panic!("Expected time count, got {:?}", sample.value);
+        };
+        assert!(*total > 0.0);
+        let labels = sorted_prometheus_metric_labels(&sample.labels);
+        insta::assert_json_snapshot!(labels);
     }
 }

src/observability/snapshots/axum_graphql__observability__metrics__tests__metrics_endpoint_returns_collected_metrics-2.snap

@@ -0,0 +1,10 @@
+---
+source: src/observability/metrics.rs
+expression: labels
+snapshot_kind: text
+---
+{
+  "method": "GET",
+  "path": "/health",
+  "status": "200"
+}

src/observability/snapshots/axum_graphql__observability__metrics__tests__metrics_endpoint_returns_collected_metrics-3.snap

@@ -0,0 +1,10 @@
+---
+source: src/observability/metrics.rs
+expression: labels
+snapshot_kind: text
+---
+{
+  "method": "GET",
+  "path": "/health",
+  "status": "200"
+}

src/observability/snapshots/axum_graphql__observability__metrics__tests__metrics_endpoint_returns_collected_metrics-4.snap

@@ -0,0 +1,10 @@
+---
+source: src/observability/metrics.rs
+expression: labels
+snapshot_kind: text
+---
+{
+  "method": "GET",
+  "path": "/health",
+  "status": "200"
+}

src/observability/snapshots/axum_graphql__observability__metrics__tests__metrics_endpoint_returns_collected_metrics.snap

@@ -0,0 +1,10 @@
+---
+source: src/observability/metrics.rs
+expression: labels
+snapshot_kind: text
+---
+{
+  "method": "GET",
+  "path": "/health",
+  "status": "200"
+}