diff --git a/Pipfile b/Pipfile index 10c54c4..f3664ba 100644 --- a/Pipfile +++ b/Pipfile @@ -28,6 +28,7 @@ requests-mock = "*" freezegun = "*" sqlalchemy-stubs = "*" boto3 = "*" +pytest-mock = "*" [requires] python_version = "3.11" diff --git a/Pipfile.lock b/Pipfile.lock index d7496f7..88cedf8 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d42ddcf11bbd3fefba6d379c9f9787c945f252ecec9acdf018a4a56074ef08e0" + "sha256": "f5d1453c2db4e24b390883065c78b6cdb8e5a6c1669f6e4170a41d669949a877" }, "pipfile-spec": 6, "requires": { @@ -439,7 +439,7 @@ "sha256:fbd6acc766814ea6443628f4e6751d0da6593dae29c08c0b2606164db026970c", "sha256:feff59f27338135776f6d4e2ec7aeeac5d5f7a08a83e80869121ef8164b74af9" ], - "markers": "python_version < '3.12' and python_version >= '3.9'", + "markers": "python_version == '3.11'", "version": "==2.0.0" }, "oracledb": { @@ -567,12 +567,12 @@ }, "sentry-sdk": { "hashes": [ - "sha256:6051562d2cfa8087bb8b4b8b79dc44690f8a054762a29c07e22588b1f619bfb5", - "sha256:aa4314f877d9cd9add5a0c9ba18e3f27f99f7de835ce36bd150e48a41c7c646f" + "sha256:0bea5fa8b564cc0d09f2e6f55893e8f70286048b0ffb3a341d5b695d1af0e6ee", + "sha256:4c85bad74df9767976afb3eeddc33e0e153300e887d637775a753a35ef99bee6" ], "index": "pypi", "markers": "python_version >= '3.6'", - "version": "==2.8.0" + "version": "==2.9.0" }, "setuptools": { "hashes": [ @@ -759,20 +759,20 @@ }, "boto3": { "hashes": [ - "sha256:72daee953cfa0631c584c9e3aef594079e1fe6a2f64c81ff791dab9a7b25c013", - "sha256:cae11cb54f79795e44248a9e53ec5c7328519019df1ba54bc01413f51c548626" + "sha256:0d16832f23e6bd3ae94e35ea8e625529850bfad9baccd426de96ad8f445d8e03", + "sha256:b590ce80c65149194def43ebf0ea1cf0533945502507837389a8d22e3ecbcf05" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.34.142" + "version": "==1.34.143" }, "botocore": { "hashes": [ - "sha256:2eeb8e6be729c1f8ded723970ed6c6ac29cc3014d86a99e73428fa8bdca81f63", - "sha256:9d8095bab0b93b9064e856730a7ffbbb4f897353d3170bec9ddccc5f4a3753bc" + "sha256:059f032ec05733a836e04e869c5a15534420102f93116f3bc9a5b759b0651caf", + "sha256:094aea179e8aaa1bc957ad49cc27d93b189dd3a1f3075d8b0ca7c445a2a88430" ], "markers": "python_version >= '3.8'", - "version": "==1.34.142" + "version": "==1.34.143" }, "certifi": { "hashes": [ @@ -900,61 +900,61 @@ "toml" ], "hashes": [ - "sha256:018a12985185038a5b2bcafab04ab833a9a0f2c59995b3cec07e10074c78635f", - "sha256:02ff6e898197cc1e9fa375581382b72498eb2e6d5fc0b53f03e496cfee3fac6d", - "sha256:042183de01f8b6d531e10c197f7f0315a61e8d805ab29c5f7b51a01d62782747", - "sha256:1014fbf665fef86cdfd6cb5b7371496ce35e4d2a00cda501cf9f5b9e6fced69f", - "sha256:1137f46adb28e3813dec8c01fefadcb8c614f33576f672962e323b5128d9a68d", - "sha256:16852febd96acd953b0d55fc842ce2dac1710f26729b31c80b940b9afcd9896f", - "sha256:2174e7c23e0a454ffe12267a10732c273243b4f2d50d07544a91198f05c48f47", - "sha256:2214ee920787d85db1b6a0bd9da5f8503ccc8fcd5814d90796c2f2493a2f4d2e", - "sha256:3257fdd8e574805f27bb5342b77bc65578e98cbc004a92232106344053f319ba", - "sha256:3684bc2ff328f935981847082ba4fdc950d58906a40eafa93510d1b54c08a66c", - "sha256:3a6612c99081d8d6134005b1354191e103ec9705d7ba2754e848211ac8cacc6b", - "sha256:3d7564cc09dd91b5a6001754a5b3c6ecc4aba6323baf33a12bd751036c998be4", - "sha256:44da56a2589b684813f86d07597fdf8a9c6ce77f58976727329272f5a01f99f7", - "sha256:5013ed890dc917cef2c9f765c4c6a8ae9df983cd60dbb635df8ed9f4ebc9f555", - "sha256:54317c2b806354cbb2dc7ac27e2b93f97096912cc16b18289c5d4e44fc663233", - "sha256:56b4eafa21c6c175b3ede004ca12c653a88b6f922494b023aeb1e836df953ace", - "sha256:581ea96f92bf71a5ec0974001f900db495488434a6928a2ca7f01eee20c23805", - "sha256:5cd64adedf3be66f8ccee418473c2916492d53cbafbfcff851cbec5a8454b136", - "sha256:5df54843b88901fdc2f598ac06737f03d71168fd1175728054c8f5a2739ac3e4", - "sha256:65e528e2e921ba8fd67d9055e6b9f9e34b21ebd6768ae1c1723f4ea6ace1234d", - "sha256:6aae5cce399a0f065da65c7bb1e8abd5c7a3043da9dceb429ebe1b289bc07806", - "sha256:6cfb5a4f556bb51aba274588200a46e4dd6b505fb1a5f8c5ae408222eb416f99", - "sha256:7076b4b3a5f6d2b5d7f1185fde25b1e54eb66e647a1dfef0e2c2bfaf9b4c88c8", - "sha256:73ca8fbc5bc622e54627314c1a6f1dfdd8db69788f3443e752c215f29fa87a0b", - "sha256:79b356f3dd5b26f3ad23b35c75dbdaf1f9e2450b6bcefc6d0825ea0aa3f86ca5", - "sha256:7a892be37ca35eb5019ec85402c3371b0f7cda5ab5056023a7f13da0961e60da", - "sha256:8192794d120167e2a64721d88dbd688584675e86e15d0569599257566dec9bf0", - "sha256:820bc841faa502e727a48311948e0461132a9c8baa42f6b2b84a29ced24cc078", - "sha256:8f894208794b164e6bd4bba61fc98bf6b06be4d390cf2daacfa6eca0a6d2bb4f", - "sha256:a04e990a2a41740b02d6182b498ee9796cf60eefe40cf859b016650147908029", - "sha256:a44963520b069e12789d0faea4e9fdb1e410cdc4aab89d94f7f55cbb7fef0353", - "sha256:a6bb74ed465d5fb204b2ec41d79bcd28afccf817de721e8a807d5141c3426638", - "sha256:ab73b35e8d109bffbda9a3e91c64e29fe26e03e49addf5b43d85fc426dde11f9", - "sha256:aea072a941b033813f5e4814541fc265a5c12ed9720daef11ca516aeacd3bd7f", - "sha256:b1ccf5e728ccf83acd313c89f07c22d70d6c375a9c6f339233dcf792094bcbf7", - "sha256:b385d49609f8e9efc885790a5a0e89f2e3ae042cdf12958b6034cc442de428d3", - "sha256:b3d45ff86efb129c599a3b287ae2e44c1e281ae0f9a9bad0edc202179bcc3a2e", - "sha256:b4a474f799456e0eb46d78ab07303286a84a3140e9700b9e154cfebc8f527016", - "sha256:b95c3a8cb0463ba9f77383d0fa8c9194cf91f64445a63fc26fb2327e1e1eb088", - "sha256:c5986ee7ea0795a4095ac4d113cbb3448601efca7f158ec7f7087a6c705304e4", - "sha256:cdd31315fc20868c194130de9ee6bfd99755cc9565edff98ecc12585b90be882", - "sha256:cef4649ec906ea7ea5e9e796e68b987f83fa9a718514fe147f538cfeda76d7a7", - "sha256:d05c16cf4b4c2fc880cb12ba4c9b526e9e5d5bb1d81313d4d732a5b9fe2b9d53", - "sha256:d2e344d6adc8ef81c5a233d3a57b3c7d5181f40e79e05e1c143da143ccb6377d", - "sha256:d45d3cbd94159c468b9b8c5a556e3f6b81a8d1af2a92b77320e887c3e7a5d080", - "sha256:db14f552ac38f10758ad14dd7b983dbab424e731588d300c7db25b6f89e335b5", - "sha256:dbc5958cb471e5a5af41b0ddaea96a37e74ed289535e8deca404811f6cb0bc3d", - "sha256:ddbd2f9713a79e8e7242d7c51f1929611e991d855f414ca9996c20e44a895f7c", - "sha256:e16f3d6b491c48c5ae726308e6ab1e18ee830b4cdd6913f2d7f77354b33f91c8", - "sha256:e2afe743289273209c992075a5a4913e8d007d569a406ffed0bd080ea02b0633", - "sha256:e564c2cf45d2f44a9da56f4e3a26b2236504a496eb4cb0ca7221cd4cc7a9aca9", - "sha256:ed550e7442f278af76d9d65af48069f1fb84c9f745ae249c1a183c1e9d1b025c" + "sha256:0086cd4fc71b7d485ac93ca4239c8f75732c2ae3ba83f6be1c9be59d9e2c6382", + "sha256:01c322ef2bbe15057bc4bf132b525b7e3f7206f071799eb8aa6ad1940bcf5fb1", + "sha256:03cafe82c1b32b770a29fd6de923625ccac3185a54a5e66606da26d105f37dac", + "sha256:044a0985a4f25b335882b0966625270a8d9db3d3409ddc49a4eb00b0ef5e8cee", + "sha256:07ed352205574aad067482e53dd606926afebcb5590653121063fbf4e2175166", + "sha256:0d1b923fc4a40c5832be4f35a5dab0e5ff89cddf83bb4174499e02ea089daf57", + "sha256:0e7b27d04131c46e6894f23a4ae186a6a2207209a05df5b6ad4caee6d54a222c", + "sha256:1fad32ee9b27350687035cb5fdf9145bc9cf0a094a9577d43e909948ebcfa27b", + "sha256:289cc803fa1dc901f84701ac10c9ee873619320f2f9aff38794db4a4a0268d51", + "sha256:3c59105f8d58ce500f348c5b56163a4113a440dad6daa2294b5052a10db866da", + "sha256:46c3d091059ad0b9c59d1034de74a7f36dcfa7f6d3bde782c49deb42438f2450", + "sha256:482855914928c8175735a2a59c8dc5806cf7d8f032e4820d52e845d1f731dca2", + "sha256:49c76cdfa13015c4560702574bad67f0e15ca5a2872c6a125f6327ead2b731dd", + "sha256:4b03741e70fb811d1a9a1d75355cf391f274ed85847f4b78e35459899f57af4d", + "sha256:4bea27c4269234e06f621f3fac3925f56ff34bc14521484b8f66a580aacc2e7d", + "sha256:4d5fae0a22dc86259dee66f2cc6c1d3e490c4a1214d7daa2a93d07491c5c04b6", + "sha256:543ef9179bc55edfd895154a51792b01c017c87af0ebaae092720152e19e42ca", + "sha256:54dece71673b3187c86226c3ca793c5f891f9fc3d8aa183f2e3653da18566169", + "sha256:6379688fb4cfa921ae349c76eb1a9ab26b65f32b03d46bb0eed841fd4cb6afb1", + "sha256:65fa405b837060db569a61ec368b74688f429b32fa47a8929a7a2f9b47183713", + "sha256:6616d1c9bf1e3faea78711ee42a8b972367d82ceae233ec0ac61cc7fec09fa6b", + "sha256:6fe885135c8a479d3e37a7aae61cbd3a0fb2deccb4dda3c25f92a49189f766d6", + "sha256:7221f9ac9dad9492cecab6f676b3eaf9185141539d5c9689d13fd6b0d7de840c", + "sha256:76d5f82213aa78098b9b964ea89de4617e70e0d43e97900c2778a50856dac605", + "sha256:7792f0ab20df8071d669d929c75c97fecfa6bcab82c10ee4adb91c7a54055463", + "sha256:831b476d79408ab6ccfadaaf199906c833f02fdb32c9ab907b1d4aa0713cfa3b", + "sha256:9146579352d7b5f6412735d0f203bbd8d00113a680b66565e205bc605ef81bc6", + "sha256:9cc44bf0315268e253bf563f3560e6c004efe38f76db03a1558274a6e04bf5d5", + "sha256:a73d18625f6a8a1cbb11eadc1d03929f9510f4131879288e3f7922097a429f63", + "sha256:a8659fd33ee9e6ca03950cfdcdf271d645cf681609153f218826dd9805ab585c", + "sha256:a94925102c89247530ae1dab7dc02c690942566f22e189cbd53579b0693c0783", + "sha256:ad4567d6c334c46046d1c4c20024de2a1c3abc626817ae21ae3da600f5779b44", + "sha256:b2e16f4cd2bc4d88ba30ca2d3bbf2f21f00f382cf4e1ce3b1ddc96c634bc48ca", + "sha256:bbdf9a72403110a3bdae77948b8011f644571311c2fb35ee15f0f10a8fc082e8", + "sha256:beb08e8508e53a568811016e59f3234d29c2583f6b6e28572f0954a6b4f7e03d", + "sha256:c4cbe651f3904e28f3a55d6f371203049034b4ddbce65a54527a3f189ca3b390", + "sha256:c7b525ab52ce18c57ae232ba6f7010297a87ced82a2383b1afd238849c1ff933", + "sha256:ca5d79cfdae420a1d52bf177de4bc2289c321d6c961ae321503b2ca59c17ae67", + "sha256:cdab02a0a941af190df8782aafc591ef3ad08824f97850b015c8c6a8b3877b0b", + "sha256:d17c6a415d68cfe1091d3296ba5749d3d8696e42c37fca5d4860c5bf7b729f03", + "sha256:d39bd10f0ae453554798b125d2f39884290c480f56e8a02ba7a6ed552005243b", + "sha256:d4b3cd1ca7cd73d229487fa5caca9e4bc1f0bca96526b922d61053ea751fe791", + "sha256:d50a252b23b9b4dfeefc1f663c568a221092cbaded20a05a11665d0dbec9b8fb", + "sha256:da8549d17489cd52f85a9829d0e1d91059359b3c54a26f28bec2c5d369524807", + "sha256:dcd070b5b585b50e6617e8972f3fbbee786afca71b1936ac06257f7e178f00f6", + "sha256:ddaaa91bfc4477d2871442bbf30a125e8fe6b05da8a0015507bfbf4718228ab2", + "sha256:df423f351b162a702c053d5dddc0fc0ef9a9e27ea3f449781ace5f906b664428", + "sha256:dff044f661f59dace805eedb4a7404c573b6ff0cdba4a524141bc63d7be5c7fd", + "sha256:e7e128f85c0b419907d1f38e616c4f1e9f1d1b37a7949f44df9a73d5da5cd53c", + "sha256:ed8d1d1821ba5fc88d4a4f45387b65de52382fa3ef1f0115a4f7a20cdfab0e94", + "sha256:f2501d60d7497fd55e391f423f965bbe9e650e9ffc3c627d5f0ac516026000b8", + "sha256:f7db0b6ae1f96ae41afe626095149ecd1b212b424626175a6633c2999eaad45b" ], "markers": "python_version >= '3.8'", - "version": "==7.5.4" + "version": "==7.6.0" }, "coveralls": { "hashes": [ @@ -1214,6 +1214,15 @@ "markers": "python_version >= '3.8'", "version": "==8.2.2" }, + "pytest-mock": { + "hashes": [ + "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f", + "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==3.14.0" + }, "python-dateutil": { "hashes": [ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", diff --git a/hrqb/base/task.py b/hrqb/base/task.py index 7e68284..9d13d6b 100644 --- a/hrqb/base/task.py +++ b/hrqb/base/task.py @@ -213,13 +213,6 @@ def target(self) -> QuickbaseTableTarget: table_name=self.table_name, ) - @property - def parse_upsert_counts(self) -> dict | None: - """Parse results of upsert via QBClient method, if target data exists from run.""" - if self.target.exists(): - return QBClient.parse_upsert_results(self.target.read()) - return None - def get_records(self) -> list[dict]: """Get Records data that will be upserted to Quickbase. diff --git a/hrqb/config.py b/hrqb/config.py index bd782ed..902dfc8 100644 --- a/hrqb/config.py +++ b/hrqb/config.py @@ -1,8 +1,10 @@ +import copy import logging import os from typing import Any import sentry_sdk +from sentry_sdk.types import Event class Config: @@ -75,6 +77,34 @@ def configure_sentry() -> str: env = os.getenv("WORKSPACE") sentry_dsn = os.getenv("SENTRY_DSN") if sentry_dsn and sentry_dsn.lower() != "none": - sentry_sdk.init(sentry_dsn, environment=env) + sentry_sdk.init( + sentry_dsn, + environment=env, + before_send=sentry_before_send_callback, + ) return f"Sentry DSN found, exceptions will be sent to Sentry with env={env}" return "No Sentry DSN found, exceptions will not be sent to Sentry" + + +def sentry_before_send_callback(event: Event, _hint: dict) -> Event: + """Callback for modifying sentry event data before sending. + + This function is difficult to mock given how it's registered with sentry_sdk.init(), + where calling another functions for the actual work allows for mocking there. + """ + return _remove_sensitive_scope_variables(event) + + +def _remove_sensitive_scope_variables(event: Event) -> Event: + """Removes sensitive data from Sentry event. + + copy.deepcopy() is used to allow testing of the original object and the returned + object separately. + """ + new_event = copy.deepcopy(event) + if "exception" in new_event: + for exc_type in new_event["exception"]["values"]: + for stacktrace in exc_type.get("stacktrace", {}).get("frames", []): + for item in ["vars", "pre_context", "post_context"]: + stacktrace.pop(item, None) + return new_event diff --git a/tests/conftest.py b/tests/conftest.py index d016c30..9cef595 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -919,3 +919,11 @@ def task_transform_years_complete( task = TransformYears(pipeline=all_tasks_pipeline_name) task.run() return task + + +@pytest.fixture +def sensitive_scope_variable(): + return { + "note": "I am a dictionary with sensitive information", + "secret": "very-secret-abc123", + } diff --git a/tests/test_base_task.py b/tests/test_base_task.py index 69053c7..4805bbb 100644 --- a/tests/test_base_task.py +++ b/tests/test_base_task.py @@ -163,31 +163,6 @@ def test_quickbase_task_run_upsert_and_json_receipt_output_target_success( assert task_load_animals.target.read() == mocked_qb_upsert_receipt -def test_quickbase_task_run_upsert_and_json_receipt_output_target_api_errors_logged( - caplog, task_transform_animals_target, task_load_animals -): - """Mocks upsert to Quickbase, asserting mocked response is written as Target data""" - mocked_qb_upsert_receipt = { - "data": [], - "metadata": { - "createdRecordIds": [11, 12], - "lineErrors": {"2": ['Incompatible value for field with ID "6".']}, - "totalNumberOfRecordsProcessed": 3, - "unchangedRecordIds": [], - "updatedRecordIds": [], - }, - } - with mock.patch("hrqb.base.task.QBClient", autospec=True) as mock_qbclient_class: - mock_qbclient = mock_qbclient_class() - mock_qbclient.get_table_id.return_value = "abcdef123" - mock_qbclient.prepare_upsert_payload.return_value = {} - mock_qbclient.upsert_records.return_value = mocked_qb_upsert_receipt - - task_load_animals.run() - - assert "errors" in task_load_animals.parse_upsert_counts - - def test_base_pipeline_name(task_pipeline_animals): assert task_pipeline_animals.pipeline_name == "Animals" diff --git a/tests/test_config.py b/tests/test_config.py index b819f8b..f75eaa8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,11 @@ +# ruff: noqa: S105, TRY301, TRY002, BLE001 + +import json import logging +import sentry_sdk + +from hrqb import config from hrqb.config import configure_logger, configure_sentry @@ -35,3 +41,28 @@ def test_configure_sentry_env_variable_is_dsn(monkeypatch): monkeypatch.setenv("SENTRY_DSN", "https://1234567890@00000.ingest.sentry.io/123456") result = configure_sentry() assert result == "Sentry DSN found, exceptions will be sent to Sentry with env=test" + + +def test_sentry_scope_variables_removed_from_sent_event( + mocker, + sensitive_scope_variable, +): + not_secret_value = "Everyone can see this exception message." + secret_value = "very-secret-abc123" + + spy_scrubber = mocker.spy(config, "_remove_sensitive_scope_variables") + try: + raise Exception(not_secret_value) + except Exception as exc: + sentry_sdk.capture_exception(exc) + + original_event = json.dumps(spy_scrubber.call_args) + scrubbed_event = json.dumps(spy_scrubber.spy_return) + + # not secret value present before and after scrubbing + assert not_secret_value in original_event + assert not_secret_value in scrubbed_event + + # secret value present in original, but absent from scrubbed + assert secret_value in original_event + assert secret_value not in scrubbed_event diff --git a/tests/test_qbclient_client.py b/tests/test_qbclient_client.py index 2190f51..cecb585 100644 --- a/tests/test_qbclient_client.py +++ b/tests/test_qbclient_client.py @@ -211,6 +211,26 @@ def test_qbclient_parse_upsert_results_response_success(qbclient, mocked_qb_api_ } +def test_qbclient_parse_upsert_results_response_captures_line_errors(qbclient): + mocked_qb_upsert_receipt = { + "data": [], + "metadata": { + "createdRecordIds": [11, 12], + "lineErrors": {"2": ['Incompatible value for field with ID "6".']}, + "totalNumberOfRecordsProcessed": 3, + "unchangedRecordIds": [], + "updatedRecordIds": [], + }, + } + assert qbclient.parse_upsert_results(mocked_qb_upsert_receipt) == { + "processed": 3, + "created": 2, + "errors": {'Incompatible value for field with ID "6".': 1}, + "updated": 0, + "unchanged": 0, + } + + def test_qbclient_parse_upsert_results_response_error_return_none(qbclient): assert qbclient.parse_upsert_results({"msg": "bad API response"}) is None