From 2dfa2a4283f3bac093bb065b7e8d8a2b9636f6cc Mon Sep 17 00:00:00 2001 From: Ken Giusti Date: Mon, 18 Nov 2024 13:56:37 -0500 Subject: [PATCH] Fixes #1621: add bi-flow octet counters to HTTP/1.x observer (#1668) --- src/decoders/http1/http1_decoder.c | 5 ----- src/observers/http1/http1_observer.c | 27 +++++++++++++++++++++++++-- tests/system_tests_http_observer.py | 17 ++++++++++++++++- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/decoders/http1/http1_decoder.c b/src/decoders/http1/http1_decoder.c index 983af2801..2a6fe2e20 100644 --- a/src/decoders/http1/http1_decoder.c +++ b/src/decoders/http1/http1_decoder.c @@ -111,11 +111,6 @@ struct h1_decode_request_state_t { qd_http1_decoder_connection_t *hconn; uintptr_t user_context; -#if 0 // TBD - uint64_t client_octets; // # total octets arriving from client for this request - uint64_t server_octets; // # total octets arriving from server for this response -#endif - int32_t response_code; // sent by server bool head_request:1; // HEAD method diff --git a/src/observers/http1/http1_observer.c b/src/observers/http1/http1_observer.c index 479f4afd6..64fc6a5fc 100644 --- a/src/observers/http1/http1_observer.c +++ b/src/observers/http1/http1_observer.c @@ -27,7 +27,9 @@ struct http1_request_state_t { DEQ_LINKS(http1_request_state_t); vflow_record_t *vflow; - bool latency_done:1; // true: latency timing complete + uint64_t client_body_octets; // total bytes received in client request msg body + uint64_t server_body_octets; // total bytes received in server response msg body + bool latency_done:1; // true: vflow latency timing complete }; ALLOC_DECLARE(http1_request_state_t); ALLOC_DEFINE(http1_request_state_t); @@ -55,6 +57,10 @@ static int rx_request(qd_http1_decoder_connection_t *hconn, const char *method, hreq->vflow = vflow_start_record(VFLOW_RECORD_BIFLOW_APP, th->vflow); vflow_set_string(hreq->vflow, VFLOW_ATTRIBUTE_PROTOCOL, version_minor == 1 ? "HTTP/1.1" : "HTTP/1.0"); vflow_set_string(hreq->vflow, VFLOW_ATTRIBUTE_METHOD, method); + vflow_set_uint64(hreq->vflow, VFLOW_ATTRIBUTE_OCTETS, 0); + vflow_set_uint64(hreq->vflow, VFLOW_ATTRIBUTE_OCTETS_REVERSE, 0); + vflow_add_rate(hreq->vflow, VFLOW_ATTRIBUTE_OCTETS, VFLOW_ATTRIBUTE_OCTET_RATE); + vflow_add_rate(hreq->vflow, VFLOW_ATTRIBUTE_OCTETS_REVERSE, VFLOW_ATTRIBUTE_OCTET_RATE_REVERSE); vflow_latency_start(hreq->vflow); hreq->latency_done = false; DEQ_INSERT_TAIL(th->http1.requests, hreq); @@ -92,6 +98,23 @@ static int rx_response(qd_http1_decoder_connection_t *hconn, uintptr_t request_c } +static int rx_body(qd_http1_decoder_connection_t *hconn, uintptr_t request_context, bool from_client, const unsigned char *body, size_t length) +{ + http1_request_state_t *hreq = (http1_request_state_t *) request_context; + assert(hreq); + + if (from_client) { + hreq->client_body_octets += length; + vflow_set_uint64(hreq->vflow, VFLOW_ATTRIBUTE_OCTETS, hreq->client_body_octets); + } else { + hreq->server_body_octets += length; + vflow_set_uint64(hreq->vflow, VFLOW_ATTRIBUTE_OCTETS_REVERSE, hreq->server_body_octets); + } + + return 0; +} + + static int transaction_complete(qd_http1_decoder_connection_t *hconn, uintptr_t request_context) { qdpo_transport_handle_t *th = (qdpo_transport_handle_t *) qd_http1_decoder_connection_get_context(hconn); @@ -133,7 +156,7 @@ static qd_http1_decoder_config_t decoder_config = { .rx_response = rx_response, // .rx_header = rx_header, // .rx_headers_done = rx_headers_done, - // .rx_body = rx_body, + .rx_body = rx_body, // .message_done = message_done, .transaction_complete = transaction_complete, .protocol_error = protocol_error diff --git a/tests/system_tests_http_observer.py b/tests/system_tests_http_observer.py index f39b05dbc..10891d992 100644 --- a/tests/system_tests_http_observer.py +++ b/tests/system_tests_http_observer.py @@ -186,6 +186,7 @@ def test_01_get(self): '-G' ] + # Note: the lengths of these files are hardcoded in the expected map below: pages = ['index.html', 't100K.html', 't10K.html', 't1K.html'] for page in pages: curl_args.append(f"http://localhost:{l_port}/{page}") @@ -201,21 +202,29 @@ def test_01_get(self): "RESULT": "200", "REASON": "OK", "PROTOCOL": "HTTP/1.1", + "OCTETS": 0, + "OCTETS_REVERSE": 45, # index.html length 'END_TIME': ANY_VALUE}), ('BIFLOW_APP', {"METHOD": "GET", "RESULT": "200", "REASON": "OK", "PROTOCOL": "HTTP/1.1", + "OCTETS": 0, + "OCTETS_REVERSE": 108803, # t100K.html length 'END_TIME': ANY_VALUE}), ('BIFLOW_APP', {"METHOD": "GET", "RESULT": "200", "REASON": "OK", + "OCTETS": 0, + "OCTETS_REVERSE": 10972, # t10K.html length "PROTOCOL": "HTTP/1.1", 'END_TIME': ANY_VALUE}), ('BIFLOW_APP', {'METHOD': "GET", 'RESULT': "200", 'REASON': "OK", 'PROTOCOL': 'HTTP/1.1', + "OCTETS": 0, + "OCTETS_REVERSE": 1188, # t1K.html length 'END_TIME': ANY_VALUE}) ] } @@ -286,7 +295,13 @@ def test_02_post(self): ('BIFLOW_APP', {'PROTOCOL': 'HTTP/1.1', 'METHOD': 'POST', 'REASON': ANY_VALUE, - 'END_TIME': ANY_VALUE}) + 'END_TIME': ANY_VALUE, + # curl sends 'Start&End' as the POST request + # message body: + 'OCTETS': 9, + # The python HTTP test server replies with + # '

Dummy CGI output!

' + 'OCTETS_REVERSE': 39}) ] } success = retry(lambda: snooper_thread.match_records(expected), delay=1)