HRQB 39 - Remove sensitive data from Sentry events

Pipfile

@@ -28,6 +28,7 @@ requests-mock = "*"
 freezegun = "*"
 sqlalchemy-stubs = "*"
 boto3 = "*"
+pytest-mock = "*"
 
 [requires]
 python_version = "3.11"

hrqb/config.py

@@ -1,8 +1,10 @@
+import copy
 import logging
 import os
 from typing import Any
 
 import sentry_sdk
+from sentry_sdk.types import Event
 
 
 class Config:
@@ -75,6 +77,34 @@ def configure_sentry() -> str:
     env = os.getenv("WORKSPACE")
     sentry_dsn = os.getenv("SENTRY_DSN")
     if sentry_dsn and sentry_dsn.lower() != "none":
-        sentry_sdk.init(sentry_dsn, environment=env)
+        sentry_sdk.init(
+            sentry_dsn,
+            environment=env,
+            before_send=sentry_before_send_callback,
+        )
         return f"Sentry DSN found, exceptions will be sent to Sentry with env={env}"
     return "No Sentry DSN found, exceptions will not be sent to Sentry"
+
+
+def sentry_before_send_callback(event: Event, _hint: dict) -> Event:
+    """Callback for modifying sentry event data before sending.
+
+    This function is difficult to mock given how it's registered with sentry_sdk.init(),
+    where calling another functions for the actual work allows for mocking there.
+    """
+    return _remove_sensitive_scope_variables(event)
+
+
+def _remove_sensitive_scope_variables(event: Event) -> Event:
+    """Removes sensitive data from Sentry event.
+
+    copy.deepcopy() is used to allow testing of the original object and the returned
+    object separately.
+    """
+    new_event = copy.deepcopy(event)
+    if "exception" in new_event:
+        for exc_type in new_event["exception"]["values"]:
+            for stacktrace in exc_type.get("stacktrace", {}).get("frames", []):
+                for item in ["vars", "pre_context", "post_context"]:
+                    stacktrace.pop(item, None)
+    return new_event

tests/conftest.py

@@ -919,3 +919,11 @@ def task_transform_years_complete(
     task = TransformYears(pipeline=all_tasks_pipeline_name)
     task.run()
     return task
+
+
+@pytest.fixture
+def sensitive_scope_variable():
+    return {
+        "note": "I am a dictionary with sensitive information",
+        "secret": "very-secret-abc123",
+    }

tests/test_config.py

@@ -1,5 +1,11 @@
+# ruff: noqa: S105, TRY301, TRY002, BLE001
+
+import json
 import logging
 
+import sentry_sdk
+
+from hrqb import config
 from hrqb.config import configure_logger, configure_sentry
 
 
@@ -35,3 +41,28 @@ def test_configure_sentry_env_variable_is_dsn(monkeypatch):
     monkeypatch.setenv("SENTRY_DSN", "https://1234567890@00000.ingest.sentry.io/123456")
     result = configure_sentry()
     assert result == "Sentry DSN found, exceptions will be sent to Sentry with env=test"
+
+
+def test_sentry_scope_variables_removed_from_sent_event(
+    mocker,
+    sensitive_scope_variable,
+):
+    not_secret_value = "Everyone can see this exception message."
+    secret_value = "very-secret-abc123"
+
+    spy_scrubber = mocker.spy(config, "_remove_sensitive_scope_variables")
+    try:
+        raise Exception(not_secret_value)
+    except Exception as exc:
+        sentry_sdk.capture_exception(exc)
+
+    original_event = json.dumps(spy_scrubber.call_args)
+    scrubbed_event = json.dumps(spy_scrubber.spy_return)
+
+    # not secret value present before and after scrubbing
+    assert not_secret_value in original_event
+    assert not_secret_value in scrubbed_event
+
+    # secret value present in original, but absent from scrubbed
+    assert secret_value in original_event
+    assert secret_value not in scrubbed_event