Correlation ID in Kafka Providers (GSI 506) (#76)

* Add tests for kafka

* Add correlation_id to kafka providers

* Pull in template update for util script formatting

* Bring kafka unit tests up to speed

* Make mypy happy

* Update lock files and pre-commit hook revs

* Add cid header to stream calc integration test

* Bump version from 1.1.0 -> 1.2.0

* Delete topics after each run in kafka cid test

Trying to figure out why it fails on GH and not locally

* Use default fixture scope in correlation tests

* Cap pytest-asyncio to below 0.23.0 for now

* Simplify cid eval in _publish_validated

* Provide a name for the correlation.py logger

* Fix documentation blips and remove an unneeded var

---------

Co-authored-by: TheByronHimes <TheByronHimes@gmail.com>

.pre-commit-config.yaml

@@ -51,7 +51,7 @@ repos:
       - id: debug-statements
       - id: debug-statements
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.6
+    rev: v0.1.7
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

examples/stream_calc/sc_tests/integration/test_event_api.py

@@ -34,7 +34,12 @@
 from stream_calc.config import Config
 from stream_calc.main import main
 
-DEFAULT_CONFIG = Config()  # type: ignore
+DEFAULT_CONFIG = Config()
+VALID_CORRELATION_ID = "7041eb31-7333-4b57-97d7-90f5562c3383"
+CORRELATION_ID_HEADER = (
+    "correlation_id",
+    bytes(VALID_CORRELATION_ID, encoding="ascii"),
+)
 
 
 class Event(NamedTuple):
@@ -136,7 +141,10 @@ def submit_test_problems(
             topic=topic,
             value=case.problem.payload,
             key="test_examples",
-            headers=[("type", case.problem.type_.encode("ascii"))],
+            headers=[
+                ("type", case.problem.type_.encode("ascii")),
+                CORRELATION_ID_HEADER,
+            ],
         )
 
     producer.flush()
@@ -208,7 +216,7 @@ async def test_receive_calc_publish(cases: list[Case] = deepcopy(CASES)):
 
         # run the stream_calc app:
         # (for each problem separately to avoid running forever)
-        config = Config(kafka_servers=[kafka_server])  # type: ignore
+        config = Config(kafka_servers=[kafka_server])
         for _ in cases:
             await main(config=config, run_forever=False)
 

examples/stream_calc/stream_calc/main.py

@@ -40,7 +40,7 @@ def get_container(config: Config) -> Container:
 
 async def main(
     *,
-    config: Config = Config(),  # type: ignore [call-arg]
+    config: Config = Config(),
     run_forever: bool = True,
 ) -> None:
     """

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hexkit"
-version = "1.1.0"
+version = "1.2.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 readme = "README.md"
 authors = [

requirements-dev.in

@@ -5,3 +5,4 @@
 
 # additional requirements can be listed her
 cryptography >= 41
+pytest-asyncio < 0.23.0

requirements-dev.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --all-extras --generate-hashes --output-file=/workspace/requirements-dev.txt /tmp/tmpkydq5tf9/pyproject.toml /workspace/requirements-dev.in
+#    pip-compile --all-extras --generate-hashes --output-file=/workspace/requirements-dev.txt /tmp/tmpmgcrq26u/pyproject.toml /workspace/requirements-dev.in
 #
 aiokafka==0.8.1 \
     --hash=sha256:1e24839088fd6d3ff481cc09a48ea487b997328df11630bc0a1b88255edbcfe9 \
@@ -49,13 +49,13 @@ attrs==23.1.0 \
     # via
     #   jsonschema
     #   referencing
-boto3==1.33.4 \
-    --hash=sha256:7668aef805fe53e487e3742cad0ffffe5fb62c3752514dd28ef81bad5ff0cb38 \
-    --hash=sha256:f26391fce6fec478fa1ab92f1ea80e4d057dd6240cc6fbe4e38fb3fce4cc8026
+boto3==1.33.9 \
+    --hash=sha256:9486f66f9a89f66d64d25cb4baa55aad323a1734ef9815b7a2c4c0787a3fc1fb \
+    --hash=sha256:baa6ea61527bcc82365a4ef365aa8f34126483ff8533d01274f3cdb340c22d73
     # via hexkit (pyproject.toml)
-botocore==1.33.4 \
-    --hash=sha256:872decbc760c3b2942477cda905d4443bd8a97511dcee3e9ca09eeb9299ad5e2 \
-    --hash=sha256:bbd96c211b670d17191d617423f3c4c277964eeb586a33b3759691c33904629d
+botocore==1.33.9 \
+    --hash=sha256:07ff4eea62f546da540310887000c81fca802c788eb374fef06eda28009dcafd \
+    --hash=sha256:1b90be11dd9a9a25290c995a6b713e76b9e2bf7bf86ba9036e0d28f2ceb2edfc
     # via
     #   boto3
     #   hexkit (pyproject.toml)
@@ -425,9 +425,9 @@ httpx==0.25.2 \
     # via
     #   -r /workspace/requirements-dev-common.in
     #   pytest-httpx
-identify==2.5.32 \
-    --hash=sha256:0b7656ef6cba81664b783352c73f8c24b39cf82f926f78f4550eda928e5e0545 \
-    --hash=sha256:5d9979348ec1a21c768ae07e0a652924538e8bce67313a73cb0f681cf08ba407
+identify==2.5.33 \
+    --hash=sha256:161558f9fe4559e1557e1bff323e8631f6a0e4837f7497767c1782832f16b62d \
+    --hash=sha256:d40ce5fcd762817627670da8a7d8d8e65f24342d14539c59488dc603bf662e34
     # via pre-commit
 idna==3.6 \
     --hash=sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca \
@@ -436,9 +436,9 @@ idna==3.6 \
     #   anyio
     #   httpx
     #   requests
-importlib-metadata==6.8.0 \
-    --hash=sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb \
-    --hash=sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743
+importlib-metadata==7.0.0 \
+    --hash=sha256:7fc841f8b8332803464e5dc1c63a2e59121f46ca186c0e2e182e80bf8c1319f7 \
+    --hash=sha256:d97503976bb81f40a193d41ee6570868479c69d5068651eb039c40d850c59d67
     # via build
 iniconfig==2.0.0 \
     --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \
@@ -454,9 +454,9 @@ jsonschema==4.20.0 \
     --hash=sha256:4f614fd46d8d61258610998997743ec5492a648b33cf478c1ddc23ed4598a5fa \
     --hash=sha256:ed6231f0429ecf966f5bc8dfef245998220549cbbcf140f913b7464c52c3b6b3
     # via hexkit (pyproject.toml)
-jsonschema-specifications==2023.11.1 \
-    --hash=sha256:c9b234904ffe02f079bf91b14d79987faa685fd4b39c377a0996954c0090b9ca \
-    --hash=sha256:f596778ab612b3fd29f72ea0d990393d0540a5aab18bf0407a46632eab540779
+jsonschema-specifications==2023.11.2 \
+    --hash=sha256:9472fc4fea474cd74bea4a2b190daeccb5a9e4db2ea80efcf7a1b582fc9a81b8 \
+    --hash=sha256:e74ba7c0a65e8cb49dc26837d6cfe576557084a8b423ed16a420984228104f93
     # via jsonschema
 jsonschema2md==1.0.0 \
     --hash=sha256:01e18693c8dcdd643aa79ce757c5896877c590493422d23fd0bd76562fbc424b \
@@ -524,9 +524,9 @@ pip-tools==7.3.0 \
     --hash=sha256:8717693288720a8c6ebd07149c93ab0be1fced0b5191df9e9decd3263e20d85e \
     --hash=sha256:8e9c99127fe024c025b46a0b2d15c7bd47f18f33226cf7330d35493663fc1d1d
     # via -r /workspace/requirements-dev-common.in
-platformdirs==4.0.0 \
-    --hash=sha256:118c954d7e949b35437270383a3f2531e99dd93cf7ce4dc8340d3356d30f173b \
-    --hash=sha256:cb633b2bcf10c51af60beb0ab06d2f1d69064b43abf4c185ca6b28865f3f9731
+platformdirs==4.1.0 \
+    --hash=sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380 \
+    --hash=sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420
     # via virtualenv
 pluggy==1.3.0 \
     --hash=sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12 \
@@ -758,7 +758,9 @@ pytest==7.4.3 \
 pytest-asyncio==0.21.1 \
     --hash=sha256:40a7eae6dded22c7b604986855ea48400ab15b069ae38116e8c01238e9eeb64d \
     --hash=sha256:8666c1c8ac02631d7c51ba282e0c69a8a452b211ffedf2599099845da5c5c37b
-    # via -r /workspace/requirements-dev-common.in
+    # via
+    #   -r /workspace/requirements-dev-common.in
+    #   -r /workspace/requirements-dev.in
 pytest-cov==4.1.0 \
     --hash=sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6 \
     --hash=sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a
@@ -949,24 +951,24 @@ rpds-py==0.13.2 \
     # via
     #   jsonschema
     #   referencing
-ruff==0.1.6 \
-    --hash=sha256:03910e81df0d8db0e30050725a5802441c2022ea3ae4fe0609b76081731accbc \
-    --hash=sha256:05991ee20d4ac4bb78385360c684e4b417edd971030ab12a4fbd075ff535050e \
-    --hash=sha256:137852105586dcbf80c1717facb6781555c4e99f520c9c827bd414fac67ddfb6 \
-    --hash=sha256:1610e14750826dfc207ccbcdd7331b6bd285607d4181df9c1c6ae26646d6848a \
-    --hash=sha256:1b09f29b16c6ead5ea6b097ef2764b42372aebe363722f1605ecbcd2b9207184 \
-    --hash=sha256:1cf5f701062e294f2167e66d11b092bba7af6a057668ed618a9253e1e90cfd76 \
-    --hash=sha256:3a0cd909d25f227ac5c36d4e7e681577275fb74ba3b11d288aff7ec47e3ae745 \
-    --hash=sha256:4558b3e178145491e9bc3b2ee3c4b42f19d19384eaa5c59d10acf6e8f8b57e33 \
-    --hash=sha256:491262006e92f825b145cd1e52948073c56560243b55fb3b4ecb142f6f0e9543 \
-    --hash=sha256:5c549ed437680b6105a1299d2cd30e4964211606eeb48a0ff7a93ef70b902248 \
-    --hash=sha256:683aa5bdda5a48cb8266fcde8eea2a6af4e5700a392c56ea5fb5f0d4bfdc0240 \
-    --hash=sha256:87455a0c1f739b3c069e2f4c43b66479a54dea0276dd5d4d67b091265f6fd1dc \
-    --hash=sha256:88b8cdf6abf98130991cbc9f6438f35f6e8d41a02622cc5ee130a02a0ed28703 \
-    --hash=sha256:bd98138a98d48a1c36c394fd6b84cd943ac92a08278aa8ac8c0fdefcf7138f35 \
-    --hash=sha256:e8fd1c62a47aa88a02707b5dd20c5ff20d035d634aa74826b42a1da77861b5ff \
-    --hash=sha256:ea284789861b8b5ca9d5443591a92a397ac183d4351882ab52f6296b4fdd5462 \
-    --hash=sha256:fd89b45d374935829134a082617954120d7a1470a9f0ec0e7f3ead983edc48cc
+ruff==0.1.7 \
+    --hash=sha256:0683b7bfbb95e6df3c7c04fe9d78f631f8e8ba4868dfc932d43d690698057e2e \
+    --hash=sha256:1ea109bdb23c2a4413f397ebd8ac32cb498bee234d4191ae1a310af760e5d287 \
+    --hash=sha256:276a89bcb149b3d8c1b11d91aa81898fe698900ed553a08129b38d9d6570e717 \
+    --hash=sha256:290ecab680dce94affebefe0bbca2322a6277e83d4f29234627e0f8f6b4fa9ce \
+    --hash=sha256:416dfd0bd45d1a2baa3b1b07b1b9758e7d993c256d3e51dc6e03a5e7901c7d80 \
+    --hash=sha256:45b38c3f8788a65e6a2cab02e0f7adfa88872696839d9882c13b7e2f35d64c5f \
+    --hash=sha256:4af95fd1d3b001fc41325064336db36e3d27d2004cdb6d21fd617d45a172dd96 \
+    --hash=sha256:69a4bed13bc1d5dabf3902522b5a2aadfebe28226c6269694283c3b0cecb45fd \
+    --hash=sha256:6b05e3b123f93bb4146a761b7a7d57af8cb7384ccb2502d29d736eaade0db519 \
+    --hash=sha256:6c64cb67b2025b1ac6d58e5ffca8f7b3f7fd921f35e78198411237e4f0db8e73 \
+    --hash=sha256:7f80496854fdc65b6659c271d2c26e90d4d401e6a4a31908e7e334fab4645aac \
+    --hash=sha256:8b0c2de9dd9daf5e07624c24add25c3a490dbf74b0e9bca4145c632457b3b42a \
+    --hash=sha256:90c958fe950735041f1c80d21b42184f1072cc3975d05e736e8d66fc377119ea \
+    --hash=sha256:9dcc6bb2f4df59cb5b4b40ff14be7d57012179d69c6565c1da0d1f013d29951b \
+    --hash=sha256:de02ca331f2143195a712983a57137c5ec0f10acc4aa81f7c1f86519e52b92a1 \
+    --hash=sha256:df2bb4bb6bbe921f6b4f5b6fdd8d8468c940731cb9406f274ae8c5ed7a78c478 \
+    --hash=sha256:dffd699d07abf54833e5f6cc50b85a6ff043715da8788c4a79bcd4ab4734d306
     # via -r /workspace/requirements-dev-common.in
 s3transfer==0.8.2 \
     --hash=sha256:368ac6876a9e9ed91f6bc86581e319be08188dc60d50e0d56308ed5765446283 \
@@ -1045,13 +1047,13 @@ urllib3==1.26.18 \
     #   botocore
     #   docker
     #   requests
-virtualenv==20.24.7 \
-    --hash=sha256:69050ffb42419c91f6c1284a7b24e0475d793447e35929b488bf6a0aade39353 \
-    --hash=sha256:a18b3fd0314ca59a2e9f4b556819ed07183b3e9a3702ecfe213f593d44f7b3fd
+virtualenv==20.25.0 \
+    --hash=sha256:4238949c5ffe6876362d9c0180fc6c3a824a7b12b80604eeb8085f2ed7460de3 \
+    --hash=sha256:bf51c0d9c7dd63ea8e44086fa1e4fb1093a31e963b86959257378aef020e1f1b
     # via pre-commit
-websocket-client==1.6.4 \
-    --hash=sha256:084072e0a7f5f347ef2ac3d8698a5e0b4ffbfcab607628cadabc650fc9a83a24 \
-    --hash=sha256:b3324019b3c28572086c4a319f91d1dcd44e6e11cd340232978c684a7650d0df
+websocket-client==1.7.0 \
+    --hash=sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6 \
+    --hash=sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588
     # via docker
 wheel==0.42.0 \
     --hash=sha256:177f9c9b0d45c47873b619f5b650346d632cdc35fb5e4d25058e09c9e581433d \

requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --all-extras --constraint=/workspace/requirements-dev.txt --generate-hashes --output-file=/workspace/requirements.txt /tmp/tmpkydq5tf9/pyproject.toml
+#    pip-compile --all-extras --constraint=/workspace/requirements-dev.txt --generate-hashes --output-file=/workspace/requirements.txt /tmp/tmpmgcrq26u/pyproject.toml
 #
 aiokafka==0.8.1 \
     --hash=sha256:1e24839088fd6d3ff481cc09a48ea487b997328df11630bc0a1b88255edbcfe9 \
@@ -52,15 +52,15 @@ attrs==23.1.0 \
     #   -c /workspace/requirements-dev.txt
     #   jsonschema
     #   referencing
-boto3==1.33.4 \
-    --hash=sha256:7668aef805fe53e487e3742cad0ffffe5fb62c3752514dd28ef81bad5ff0cb38 \
-    --hash=sha256:f26391fce6fec478fa1ab92f1ea80e4d057dd6240cc6fbe4e38fb3fce4cc8026
+boto3==1.33.9 \
+    --hash=sha256:9486f66f9a89f66d64d25cb4baa55aad323a1734ef9815b7a2c4c0787a3fc1fb \
+    --hash=sha256:baa6ea61527bcc82365a4ef365aa8f34126483ff8533d01274f3cdb340c22d73
     # via
     #   -c /workspace/requirements-dev.txt
     #   hexkit (pyproject.toml)
-botocore==1.33.4 \
-    --hash=sha256:872decbc760c3b2942477cda905d4443bd8a97511dcee3e9ca09eeb9299ad5e2 \
-    --hash=sha256:bbd96c211b670d17191d617423f3c4c277964eeb586a33b3759691c33904629d
+botocore==1.33.9 \
+    --hash=sha256:07ff4eea62f546da540310887000c81fca802c788eb374fef06eda28009dcafd \
+    --hash=sha256:1b90be11dd9a9a25290c995a6b713e76b9e2bf7bf86ba9036e0d28f2ceb2edfc
     # via
     #   -c /workspace/requirements-dev.txt
     #   boto3
@@ -277,9 +277,9 @@ jsonschema==4.20.0 \
     # via
     #   -c /workspace/requirements-dev.txt
     #   hexkit (pyproject.toml)
-jsonschema-specifications==2023.11.1 \
-    --hash=sha256:c9b234904ffe02f079bf91b14d79987faa685fd4b39c377a0996954c0090b9ca \
-    --hash=sha256:f596778ab612b3fd29f72ea0d990393d0540a5aab18bf0407a46632eab540779
+jsonschema-specifications==2023.11.2 \
+    --hash=sha256:9472fc4fea474cd74bea4a2b190daeccb5a9e4db2ea80efcf7a1b582fc9a81b8 \
+    --hash=sha256:e74ba7c0a65e8cb49dc26837d6cfe576557084a8b423ed16a420984228104f93
     # via
     #   -c /workspace/requirements-dev.txt
     #   jsonschema
@@ -729,9 +729,9 @@ urllib3==1.26.18 \
     #   botocore
     #   docker
     #   requests
-websocket-client==1.6.4 \
-    --hash=sha256:084072e0a7f5f347ef2ac3d8698a5e0b4ffbfcab607628cadabc650fc9a83a24 \
-    --hash=sha256:b3324019b3c28572086c4a319f91d1dcd44e6e11cd340232978c684a7650d0df
+websocket-client==1.7.0 \
+    --hash=sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6 \
+    --hash=sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588
     # via
     #   -c /workspace/requirements-dev.txt
     #   docker

scripts/list_outdated_dependencies.py

@@ -51,7 +51,7 @@ def get_main_deps_pyproject(modified_pyproject: dict[str, Any]) -> list[Requirem
 
 
 def get_optional_deps_pyproject(
-    modified_pyproject: dict[str, Any]
+    modified_pyproject: dict[str, Any],
 ) -> list[Requirement]:
     """Get a list of the optional dependencies from pyproject.toml"""
 

src/hexkit/correlation.py

@@ -23,7 +23,7 @@
 
 from hexkit.utils import set_context_var
 
-log = logging.getLogger()
+log = logging.getLogger(__name__)
 
 correlation_id_var: ContextVar[str] = ContextVar("correlation_id", default="")
 

src/hexkit/inject.py

@@ -58,7 +58,7 @@ class AsyncInitShutdownError(TypeError):
 
 
 def assert_async_constructable(
-    constructable: Union[type[AsyncContextConstructable], type[AsyncConstructable]]
+    constructable: Union[type[AsyncContextConstructable], type[AsyncConstructable]],
 ):
     """
     Make sure that the provided object has a callable attribute `construct`.