From 06acc1a326957c2daa3ffd6d636c32858d098cbb Mon Sep 17 00:00:00 2001 From: Graham Hukill Date: Thu, 2 May 2024 12:39:12 -0400 Subject: [PATCH] Establish Quickbase API Client Why these changes are being introduced: One of the primary functions of this app is to load data into Quickbase via its API. The API has some awkward edges, like the need to upsert data with Field IDs vs Field names, or the need to use Table IDs instead of names. Therefore to upsert data, you might need 2-3 API calls in advance just to map the Table and Field IDs to names. How this addresses that need: * Creates new QBClient class * QBClient has method for making API calls with authorization * QBClient caches API calls when the call signature is identical * QBClient has convenience methods for common API calls like getting Table or Field information * QBClient has some methods to map data * QBClient will be the workhorse of most Load Tasks that get built Side effects of this change: * None Relevant ticket(s): * https://mitlibraries.atlassian.net/browse/HRQB-12 --- Pipfile | 4 + Pipfile.lock | 269 +++++++++++++----- README.md | 1 + hrqb/base/task.py | 29 ++ hrqb/config.py | 2 + hrqb/exceptions.py | 5 + hrqb/utils/quickbase.py | 144 ++++++++++ tests/conftest.py | 110 ++++++- tests/fixtures/qb_api_responses/getApp.json | 19 ++ .../qb_api_responses/getAppTables.json | 38 +++ .../fixtures/qb_api_responses/getFields.json | 81 ++++++ tests/fixtures/qb_api_responses/upsert.json | 39 +++ tests/test_qbclient_client.py | 100 +++++++ 13 files changed, 776 insertions(+), 65 deletions(-) create mode 100644 hrqb/exceptions.py create mode 100644 hrqb/utils/quickbase.py create mode 100644 tests/fixtures/qb_api_responses/getApp.json create mode 100644 tests/fixtures/qb_api_responses/getAppTables.json create mode 100644 tests/fixtures/qb_api_responses/getFields.json create mode 100644 tests/fixtures/qb_api_responses/upsert.json create mode 100644 tests/test_qbclient_client.py diff --git a/Pipfile b/Pipfile index 2ab5d07..cbf5c13 100644 --- a/Pipfile +++ b/Pipfile @@ -10,6 +10,9 @@ oracledb = "*" luigi = "*" pandas = "*" pandas-stubs = "*" +attrs = "*" +requests = "*" +types-requests = "*" [dev-packages] black = "*" @@ -19,6 +22,7 @@ mypy = "*" pre-commit = "*" pytest = "*" ruff = "*" +requests-mock = "*" [requires] python_version = "3.11" diff --git a/Pipfile.lock b/Pipfile.lock index 71b6876..14c1b9e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "e94095672a2ac97f7593e1d809fb0945400e553fb00af2de964a9c70f082da3d" + "sha256": "9750b0343d4bc0c1401f574c0cf3171ec35fef343da3d21545155589e5e8946d" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,14 @@ ] }, "default": { + "attrs": { + "hashes": [ + "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30", + "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1" + ], + "index": "pypi", + "version": "==23.2.0" + }, "certifi": { "hashes": [ "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f", @@ -82,6 +90,102 @@ "markers": "platform_python_implementation != 'PyPy'", "version": "==1.16.0" }, + "charset-normalizer": { + "hashes": [ + "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", + "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", + "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", + "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", + "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09", + "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", + "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", + "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", + "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", + "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", + "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", + "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", + "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", + "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", + "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", + "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", + "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73", + "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", + "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", + "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", + "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", + "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", + "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", + "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", + "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", + "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", + "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", + "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", + "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", + "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", + "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77", + "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", + "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", + "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", + "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", + "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", + "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25", + "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", + "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab", + "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", + "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", + "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", + "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", + "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", + "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99", + "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", + "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", + "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", + "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", + "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", + "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", + "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", + "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", + "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", + "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001", + "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", + "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", + "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", + "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", + "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", + "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", + "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f", + "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d", + "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", + "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", + "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", + "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", + "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", + "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", + "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", + "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", + "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", + "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", + "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", + "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7", + "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4", + "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", + "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", + "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", + "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", + "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", + "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", + "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", + "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b", + "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", + "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", + "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", + "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", + "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", + "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==3.3.2" + }, "click": { "hashes": [ "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", @@ -136,6 +240,14 @@ "markers": "python_version >= '3.9'", "version": "==0.21.2" }, + "idna": { + "hashes": [ + "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc", + "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0" + ], + "markers": "python_version >= '3.5'", + "version": "==3.7" + }, "lockfile": { "hashes": [ "sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799", @@ -303,6 +415,14 @@ ], "version": "==2024.1" }, + "requests": { + "hashes": [ + "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", + "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" + ], + "index": "pypi", + "version": "==2.31.0" + }, "sentry-sdk": { "hashes": [ "sha256:b54c54a2160f509cf2757260d0cf3885b608c6192c2555a3857e3a4d0f84bdb3", @@ -360,6 +480,14 @@ "markers": "python_version >= '3.8'", "version": "==2024.1.0.20240417" }, + "types-requests": { + "hashes": [ + "sha256:4428df33c5503945c74b3f42e82b181e86ec7b724620419a2966e2de604ce1a1", + "sha256:6216cdac377c6b9a040ac1c0404f7284bd13199c0e1bb235f4324627e8898cf5" + ], + "index": "pypi", + "version": "==2.31.0.20240406" + }, "tzdata": { "hashes": [ "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd", @@ -534,68 +662,73 @@ "version": "==8.1.7" }, "coverage": { - "hashes": [ - "sha256:027018943386e7b942fa832372ebc120155fd970837489896099f5cfa2890f79", - "sha256:11b990d520ea75e7ee8dcab5bc908072aaada194a794db9f6d7d5cfd19661e5a", - "sha256:12adf310e4aafddc58afdb04d686795f33f4d7a6fa67a7a9d4ce7d6ae24d949f", - "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a", - "sha256:1ef221513e6f68b69ee9e159506d583d31aa3567e0ae84eaad9d6ec1107dddaa", - "sha256:20c8ac5386253717e5ccc827caad43ed66fea0efe255727b1053a8154d952398", - "sha256:2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba", - "sha256:255758a1e3b61db372ec2736c8e2a1fdfaf563977eedbdf131de003ca5779b7d", - "sha256:265de0fa6778d07de30bcf4d9dc471c3dc4314a23a3c6603d356a3c9abc2dfcf", - "sha256:33a7da4376d5977fbf0a8ed91c4dffaaa8dbf0ddbf4c8eea500a2486d8bc4d7b", - "sha256:42eafe6778551cf006a7c43153af1211c3aaab658d4d66fa5fcc021613d02518", - "sha256:4433b90fae13f86fafff0b326453dd42fc9a639a0d9e4eec4d366436d1a41b6d", - "sha256:4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795", - "sha256:4a8dbc1f0fbb2ae3de73eb0bdbb914180c7abfbf258e90b311dcd4f585d44bd2", - "sha256:59f53f1dc5b656cafb1badd0feb428c1e7bc19b867479ff72f7a9dd9b479f10e", - "sha256:5dbec3b9095749390c09ab7c89d314727f18800060d8d24e87f01fb9cfb40b32", - "sha256:633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745", - "sha256:6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b", - "sha256:6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e", - "sha256:6d4817234349a80dbf03640cec6109cd90cba068330703fa65ddf56b60223a6d", - "sha256:723e8130d4ecc8f56e9a611e73b31219595baa3bb252d539206f7bbbab6ffc1f", - "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660", - "sha256:7b6be138d61e458e18d8e6ddcddd36dd96215edfe5f1168de0b1b32635839b62", - "sha256:7ccf362abd726b0410bf8911c31fbf97f09f8f1061f8c1cf03dfc4b6372848f6", - "sha256:83516205e254a0cb77d2d7bb3632ee019d93d9f4005de31dca0a8c3667d5bc04", - "sha256:851cf4ff24062c6aec510a454b2584f6e998cada52d4cb58c5e233d07172e50c", - "sha256:8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5", - "sha256:94e2565443291bd778421856bc975d351738963071e9b8839ca1fc08b42d4bef", - "sha256:95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc", - "sha256:97117225cdd992a9c2a5515db1f66b59db634f59d0679ca1fa3fe8da32749cae", - "sha256:98e8a10b7a314f454d9eff4216a9a94d143a7ee65018dd12442e898ee2310578", - "sha256:a1170fa54185845505fbfa672f1c1ab175446c887cce8212c44149581cf2d466", - "sha256:a6b7d95969b8845250586f269e81e5dfdd8ff828ddeb8567a4a2eaa7313460c4", - "sha256:a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91", - "sha256:af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0", - "sha256:b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4", - "sha256:b5604380f3415ba69de87a289a2b56687faa4fe04dbee0754bfcae433489316b", - "sha256:b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe", - "sha256:bc8ef5e043a2af066fa8cbfc6e708d58017024dc4345a1f9757b329a249f041b", - "sha256:c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75", - "sha256:cca4435eebea7962a52bdb216dec27215d0df64cf27fc1dd538415f5d2b9da6b", - "sha256:d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c", - "sha256:d9ecf0829c6a62b9b573c7bb6d4dcd6ba8b6f80be9ba4fc7ed50bf4ac9aecd72", - "sha256:dbdb91cd8c048c2b09eb17713b0c12a54fbd587d79adcebad543bc0cd9a3410b", - "sha256:de3001a203182842a4630e7b8d1a2c7c07ec1b45d3084a83d5d227a3806f530f", - "sha256:e07f4a4a9b41583d6eabec04f8b68076ab3cd44c20bd29332c6572dda36f372e", - "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53", - "sha256:f4f05d88d9a80ad3cac6244d36dd89a3c00abc16371769f1340101d3cb899fc3", - "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84", - "sha256:fc2af30ed0d5ae0b1abdb4ebdce598eafd5b35397d4d75deb341a614d333d987" + "extras": [ + "toml" + ], + "hashes": [ + "sha256:00838a35b882694afda09f85e469c96367daa3f3f2b097d846a7216993d37f4c", + "sha256:0513b9508b93da4e1716744ef6ebc507aff016ba115ffe8ecff744d1322a7b63", + "sha256:09c3255458533cb76ef55da8cc49ffab9e33f083739c8bd4f58e79fecfe288f7", + "sha256:09ef9199ed6653989ebbcaacc9b62b514bb63ea2f90256e71fea3ed74bd8ff6f", + "sha256:09fa497a8ab37784fbb20ab699c246053ac294d13fc7eb40ec007a5043ec91f8", + "sha256:0f9f50e7ef2a71e2fae92774c99170eb8304e3fdf9c8c3c7ae9bab3e7229c5cf", + "sha256:137eb07173141545e07403cca94ab625cc1cc6bc4c1e97b6e3846270e7e1fea0", + "sha256:1f384c3cc76aeedce208643697fb3e8437604b512255de6d18dae3f27655a384", + "sha256:201bef2eea65e0e9c56343115ba3814e896afe6d36ffd37bab783261db430f76", + "sha256:38dd60d7bf242c4ed5b38e094baf6401faa114fc09e9e6632374388a404f98e7", + "sha256:3b799445b9f7ee8bf299cfaed6f5b226c0037b74886a4e11515e569b36fe310d", + "sha256:3ea79bb50e805cd6ac058dfa3b5c8f6c040cb87fe83de10845857f5535d1db70", + "sha256:40209e141059b9370a2657c9b15607815359ab3ef9918f0196b6fccce8d3230f", + "sha256:41c9c5f3de16b903b610d09650e5e27adbfa7f500302718c9ffd1c12cf9d6818", + "sha256:54eb8d1bf7cacfbf2a3186019bcf01d11c666bd495ed18717162f7eb1e9dd00b", + "sha256:598825b51b81c808cb6f078dcb972f96af96b078faa47af7dfcdf282835baa8d", + "sha256:5fc1de20b2d4a061b3df27ab9b7c7111e9a710f10dc2b84d33a4ab25065994ec", + "sha256:623512f8ba53c422fcfb2ce68362c97945095b864cda94a92edbaf5994201083", + "sha256:690db6517f09336559dc0b5f55342df62370a48f5469fabf502db2c6d1cffcd2", + "sha256:69eb372f7e2ece89f14751fbcbe470295d73ed41ecd37ca36ed2eb47512a6ab9", + "sha256:73bfb9c09951125d06ee473bed216e2c3742f530fc5acc1383883125de76d9cd", + "sha256:742a76a12aa45b44d236815d282b03cfb1de3b4323f3e4ec933acfae08e54ade", + "sha256:7c95949560050d04d46b919301826525597f07b33beba6187d04fa64d47ac82e", + "sha256:8130a2aa2acb8788e0b56938786c33c7c98562697bf9f4c7d6e8e5e3a0501e4a", + "sha256:8a2b2b78c78293782fd3767d53e6474582f62443d0504b1554370bde86cc8227", + "sha256:8ce1415194b4a6bd0cdcc3a1dfbf58b63f910dcb7330fe15bdff542c56949f87", + "sha256:9ca28a302acb19b6af89e90f33ee3e1906961f94b54ea37de6737b7ca9d8827c", + "sha256:a4cdc86d54b5da0df6d3d3a2f0b710949286094c3a6700c21e9015932b81447e", + "sha256:aa5b1c1bfc28384f1f53b69a023d789f72b2e0ab1b3787aae16992a7ca21056c", + "sha256:aadacf9a2f407a4688d700e4ebab33a7e2e408f2ca04dbf4aef17585389eff3e", + "sha256:ae71e7ddb7a413dd60052e90528f2f65270aad4b509563af6d03d53e979feafd", + "sha256:b14706df8b2de49869ae03a5ccbc211f4041750cd4a66f698df89d44f4bd30ec", + "sha256:b1a93009cb80730c9bca5d6d4665494b725b6e8e157c1cb7f2db5b4b122ea562", + "sha256:b2991665420a803495e0b90a79233c1433d6ed77ef282e8e152a324bbbc5e0c8", + "sha256:b2c5edc4ac10a7ef6605a966c58929ec6c1bd0917fb8c15cb3363f65aa40e677", + "sha256:b4d33f418f46362995f1e9d4f3a35a1b6322cb959c31d88ae56b0298e1c22357", + "sha256:b91cbc4b195444e7e258ba27ac33769c41b94967919f10037e6355e998af255c", + "sha256:c74880fc64d4958159fbd537a091d2a585448a8f8508bf248d72112723974cbd", + "sha256:c901df83d097649e257e803be22592aedfd5182f07b3cc87d640bbb9afd50f49", + "sha256:cac99918c7bba15302a2d81f0312c08054a3359eaa1929c7e4b26ebe41e9b286", + "sha256:cc4f1358cb0c78edef3ed237ef2c86056206bb8d9140e73b6b89fbcfcbdd40e1", + "sha256:ccd341521be3d1b3daeb41960ae94a5e87abe2f46f17224ba5d6f2b8398016cf", + "sha256:ce4b94265ca988c3f8e479e741693d143026632672e3ff924f25fab50518dd51", + "sha256:cf271892d13e43bc2b51e6908ec9a6a5094a4df1d8af0bfc360088ee6c684409", + "sha256:d5ae728ff3b5401cc320d792866987e7e7e880e6ebd24433b70a33b643bb0384", + "sha256:d71eec7d83298f1af3326ce0ff1d0ea83c7cb98f72b577097f9083b20bdaf05e", + "sha256:d898fe162d26929b5960e4e138651f7427048e72c853607f2b200909794ed978", + "sha256:d89d7b2974cae412400e88f35d86af72208e1ede1a541954af5d944a8ba46c57", + "sha256:dfa8fe35a0bb90382837b238fff375de15f0dcdb9ae68ff85f7a63649c98527e", + "sha256:e0be5efd5127542ef31f165de269f77560d6cdef525fffa446de6f7e9186cfb2", + "sha256:fdfafb32984684eb03c2d83e1e51f64f0906b11e64482df3c5db936ce3839d48", + "sha256:ff7687ca3d7028d8a5f0ebae95a6e4827c5616b31a4ee1192bdfde697db110d4" ], - "markers": "python_version >= '3.7'", - "version": "==6.5.0" + "markers": "python_version >= '3.8'", + "version": "==7.4.4" }, "coveralls": { "hashes": [ - "sha256:b32a8bb5d2df585207c119d6c01567b81fba690c9c10a753bfe27a335bfc43ea", - "sha256:f42015f31d386b351d4226389b387ae173207058832fbf5c8ec4b40e27b16026" + "sha256:401715d244a27d5da03eb1ac614aa585cc7e4dd5b0d4c035113b6349da4e6161", + "sha256:9486f353176d309066053d38edbade3aad6346c5eb8a5edde7090d3116219414" ], "index": "pypi", - "version": "==3.3.1" + "version": "==4.0.0" }, "decorator": { "hashes": [ @@ -628,11 +761,11 @@ }, "filelock": { "hashes": [ - "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f", - "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4" + "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f", + "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a" ], "markers": "python_version >= '3.8'", - "version": "==3.13.4" + "version": "==3.14.0" }, "identify": { "hashes": [ @@ -887,9 +1020,17 @@ "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" ], - "markers": "python_version >= '3.7'", + "index": "pypi", "version": "==2.31.0" }, + "requests-mock": { + "hashes": [ + "sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563", + "sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401" + ], + "index": "pypi", + "version": "==1.12.1" + }, "ruff": { "hashes": [ "sha256:0e2e06459042ac841ed510196c350ba35a9b24a643e23db60d79b2db92af0c2b", @@ -962,11 +1103,11 @@ }, "virtualenv": { "hashes": [ - "sha256:0846377ea76e818daaa3e00a4365c018bc3ac9760cbb3544de542885aad61fb3", - "sha256:ec25a9671a5102c8d2657f62792a27b48f016664c6873f6beed3800008577210" + "sha256:604bfdceaeece392802e6ae48e69cec49168b9c5f4a44e483963f9242eb0e78b", + "sha256:7aa9982a728ae5892558bff6a2839c00b9ed145523ece2274fad6f414690ae75" ], "markers": "python_version >= '3.7'", - "version": "==20.26.0" + "version": "==20.26.1" }, "wcwidth": { "hashes": [ diff --git a/README.md b/README.md index a58d3d9..3185a4d 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ See additional diagrams and documentation in the [docs](docs) folder: SENTRY_DSN=# If set to a valid Sentry DSN, enables Sentry exception monitoring. This is not needed for local development. WORKSPACE=# Set to `dev` for local development, this will be set to `stage` and `prod` in those environments by Terraform. LUIGI_CONFIG_PATH=hrqb/luigi.cfg # this env var must be set, pointing to config file in hrqb folder +QUICKBASE_API_TOKEN=# Quickbase API token ``` ### Optional diff --git a/hrqb/base/task.py b/hrqb/base/task.py index 8847e68..df869a9 100644 --- a/hrqb/base/task.py +++ b/hrqb/base/task.py @@ -5,6 +5,7 @@ from hrqb.base import PandasPickleTarget, QuickbaseTableTarget from hrqb.utils import today_date +from hrqb.utils.quickbase import QBClient class HRQBTask(luigi.Task): @@ -65,6 +66,34 @@ def target(self) -> QuickbaseTableTarget: def output(self) -> QuickbaseTableTarget: return self.target() + def get_records(self) -> list[dict]: + """Get Records data that will be upserted to Quickbase. + + This method may be overridden if necessary if a load Task requires more complex + behavior than a straight conversion of the parent's DataFrame to a dictionary. + """ + return self.input_pandas_dataframe.to_dict(orient="records") + + def run(self) -> None: + """Retrieve data from parent Task and upsert to Quickbase table. + + Because Load Tasks (upserting data to Quickbase) are so uniform, this run method + can be defined on this base class. All data required for this operation exists + on the Task: data from parent Transform class and QB table name. + """ + records = self.get_records() + + qbclient = QBClient() + table_id = qbclient.get_table_id(self.table_name) + upsert_payload = qbclient.prepare_upsert_payload( + table_id, + records, + merge_field=None, + ) + results = qbclient.upsert_records(upsert_payload) + + self.target().write(results) + class HRQBPipelineTask(luigi.WrapperTask): date = luigi.DateParameter(default=today_date()) diff --git a/hrqb/config.py b/hrqb/config.py index 1960d5d..b89001f 100644 --- a/hrqb/config.py +++ b/hrqb/config.py @@ -10,6 +10,8 @@ class Config: "WORKSPACE", "SENTRY_DSN", "LUIGI_CONFIG_PATH", + "QUICKBASE_API_TOKEN", + "QUICKBASE_APP_ID", ) OPTIONAL_ENV_VARS = ("DYLD_LIBRARY_PATH",) diff --git a/hrqb/exceptions.py b/hrqb/exceptions.py new file mode 100644 index 0000000..7cc78ea --- /dev/null +++ b/hrqb/exceptions.py @@ -0,0 +1,5 @@ +"""hrqb.exceptions""" + + +class QBFieldNotFoundError(ValueError): + pass diff --git a/hrqb/utils/quickbase.py b/hrqb/utils/quickbase.py new file mode 100644 index 0000000..3fe35bd --- /dev/null +++ b/hrqb/utils/quickbase.py @@ -0,0 +1,144 @@ +"""utils.quickbase""" + +import json +import logging +from collections.abc import Callable + +import pandas as pd +import requests +from attrs import define, field + +from hrqb.config import Config +from hrqb.exceptions import QBFieldNotFoundError + +logger = logging.getLogger(__name__) + +RequestsMethod = Callable[..., requests.Response] + + +@define +class QBClient: + api_base: str = field(default="https://api.quickbase.com/v1") + cache_results: bool = field(default=True) + _cache: dict = field(factory=dict, repr=False) + + @property + def request_headers(self) -> dict: + return { + "Authorization": f"QB-USER-TOKEN {Config().QUICKBASE_API_TOKEN}", + "QB-Realm-Hostname": "mit.quickbase.com", + } + + @property + def app_id(self) -> str: + return Config().QUICKBASE_APP_ID + + def make_request( + self, requests_method: RequestsMethod, path: str, **kwargs: dict + ) -> dict: + """Make an API request to Quickbase API. + + This method caches request responses, such that data from informational requests + may be reused in later operations. + """ + # hash the request to cache the response + request_hash = (path, json.dumps(kwargs, sort_keys=True)) + if self.cache_results and request_hash in self._cache: + message = f"Using cached result for path: {path}" + logger.debug(message) + return self._cache[request_hash] + + # make API call + results = requests_method( + f"{self.api_base}/{path.removeprefix('/')}", + headers=self.request_headers, + **kwargs, + ).json() + if self.cache_results: + self._cache[request_hash] = results + + return results + + def get_app_info(self) -> dict: + """Retrieve information about the QB app. + + https://developer.quickbase.com/operation/getApp + """ + return self.make_request(requests.get, f"apps/{self.app_id}") + + def get_tables(self) -> pd.DataFrame: + """Get all QB Tables as a Dataframe. + + https://developer.quickbase.com/operation/getAppTables + """ + tables = self.make_request(requests.get, f"tables?appId={self.app_id}") + return pd.DataFrame(tables) + + def get_table_id(self, name: str) -> str: + """Get Table ID from Dataframe of Tables.""" + tables_df = self.get_tables() + return tables_df[tables_df.name == name].iloc[0].id + + def get_table_fields(self, table_id: str) -> pd.DataFrame: + """Get all QB Table Fields as a Dataframe. + + https://developer.quickbase.com/operation/getFields + """ + fields = self.make_request(requests.get, f"fields?tableId={table_id}") + return pd.DataFrame(fields) + + def get_table_fields_name_to_id(self, table_id: str) -> dict: + """Get Field name-to-id map for a Table. + + This method is particularly helpful for upserting data via the QB API, where + Field IDs are required instead of Field names. + """ + fields_df = self.get_table_fields(table_id) + return {f["label"]: f["id"] for _, f in fields_df.iterrows()} + + def upsert_records(self, upsert_payload: dict) -> dict: + """Upsert Records into a Table. + + https://developer.quickbase.com/operation/upsert + """ + return self.make_request(requests.post, "records", json=upsert_payload) + + def prepare_upsert_payload( + self, + table_id: str, + records: list[dict], + merge_field: str | None = None, + ) -> dict: + """Prepare an API payload for upsert. + + https://developer.quickbase.com/operation/upsert + + This method expects a list of dictionaries, one dictionary per record, with a + {Field Name:Value} structure. This method will first retrieve a mapping of + Field name-to-ID, then remap the data to a {Field ID:Value} structure. + + Then, return a dictionary payload suitable for the QB upsert API call. + """ + field_map = self.get_table_fields_name_to_id(table_id) + mapped_records = [] + for record in records: + mapped_record = {} + for field_name, field_value in record.items(): + if field_id := field_map.get(field_name): + mapped_record[str(field_id)] = {"value": field_value} + else: + message = ( + f"Field name '{field_name}' not found for Table ID '{table_id}'" + ) + raise QBFieldNotFoundError(message) + mapped_records.append(mapped_record) + + upsert_payload = { + "to": table_id, + "data": mapped_records, + "fieldsToReturn": list(field_map.values()), + } + if merge_field: + upsert_payload["mergeFieldId"] = field_map[merge_field] + + return upsert_payload diff --git a/tests/conftest.py b/tests/conftest.py index 8a77707..32cfdf7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,15 @@ +# ruff: noqa: N802, N803 + +import json + import pandas as pd import pytest +import requests_mock from click.testing import CliRunner -from hrqb.base.task import PandasPickleTask, QuickbaseUpsertTask +from hrqb.base import QuickbaseTableTarget +from hrqb.base.task import PandasPickleTarget, PandasPickleTask, QuickbaseUpsertTask +from hrqb.utils.quickbase import QBClient @pytest.fixture(autouse=True) @@ -10,6 +17,8 @@ def _test_env(monkeypatch): monkeypatch.setenv("SENTRY_DSN", "None") monkeypatch.setenv("WORKSPACE", "test") monkeypatch.setenv("LUIGI_CONFIG_PATH", "hrqb/luigi.cfg") + monkeypatch.setenv("QUICKBASE_API_TOKEN", "qb-api-acb123") + monkeypatch.setenv("QUICKBASE_APP_ID", "qb-app-def456") @pytest.fixture @@ -118,3 +127,102 @@ def requires(self): return [complete_first_pandas_series_task] return SecondTask(path=f"{tmpdir}/bar.pickle", table_name="bar") + + +@pytest.fixture +def qbclient(): + return QBClient() + + +@pytest.fixture(scope="session", autouse=True) +def global_requests_mock(): + with requests_mock.Mocker() as m: + yield m + + +@pytest.fixture +def mocked_qb_api_getApp(qbclient, global_requests_mock): + url = f"{qbclient.api_base}/apps/{qbclient.app_id}" + with open("tests/fixtures/qb_api_responses/getApp.json") as f: + api_response = json.load(f) + global_requests_mock.get(url, json=api_response) + return api_response + + +@pytest.fixture +def mocked_qb_api_getAppTables(qbclient, global_requests_mock): + url = f"{qbclient.api_base}/tables?appId={qbclient.app_id}" + with open("tests/fixtures/qb_api_responses/getAppTables.json") as f: + api_response = json.load(f) + global_requests_mock.get(url, json=api_response) + return api_response + + +@pytest.fixture +def mocked_table_id(): + return "bpqe82s1" + + +@pytest.fixture +def mocked_table_name(): + return "Example Table #0" + + +@pytest.fixture +def mocked_qb_api_getFields(qbclient, mocked_table_id, global_requests_mock): + url = f"{qbclient.api_base}/fields?tableId={mocked_table_id}" + with open("tests/fixtures/qb_api_responses/getFields.json") as f: + api_response = json.load(f) + global_requests_mock.get(url, json=api_response) + return api_response + + +@pytest.fixture +def mocked_upsert_data(): + return [ + {"Field1": "Green", "Numeric Field": 42}, + {"Field1": "Red", "Numeric Field": 101}, + {"Field1": "Blue", "Numeric Field": 999}, + ] + + +@pytest.fixture +def mocked_upsert_payload( + qbclient, mocked_table_id, mocked_upsert_data, mocked_qb_api_getFields +): + return qbclient.prepare_upsert_payload(mocked_table_id, mocked_upsert_data, None) + + +@pytest.fixture +def mocked_qb_api_upsert( + qbclient, mocked_table_id, mocked_upsert_payload, global_requests_mock +): + url = f"{qbclient.api_base}/records" + with open("tests/fixtures/qb_api_responses/upsert.json") as f: + api_response = json.load(f) + global_requests_mock.register_uri( + "POST", + url, + additional_matcher=lambda req: req.json() == mocked_upsert_payload, + json=api_response, + ) + return api_response + + +@pytest.fixture +def mocked_transform_pandas_target(tmpdir, mocked_table_name, mocked_upsert_data): + target = PandasPickleTarget( + path=f"{tmpdir}/transform__example_table_0.pickle", table_name=mocked_table_name + ) + target.write(pd.DataFrame(mocked_upsert_data)) + return target + + +@pytest.fixture +def quickbase_load_task_with_parent_data(mocked_transform_pandas_target): + class LoadTaskWithData(QuickbaseUpsertTask): + @property + def single_input(self) -> PandasPickleTarget | QuickbaseTableTarget: + return mocked_transform_pandas_target + + return LoadTaskWithData diff --git a/tests/fixtures/qb_api_responses/getApp.json b/tests/fixtures/qb_api_responses/getApp.json new file mode 100644 index 0000000..72771d9 --- /dev/null +++ b/tests/fixtures/qb_api_responses/getApp.json @@ -0,0 +1,19 @@ +{ + "created": "2020-03-27T18:34:12Z", + "dateFormat": "MM-DD-YYYY", + "description": "My testing app", + "hasEveryoneOnTheInternet": true, + "id": "bpqe82s1", + "name": "Testing App", + "securityProperties": { + "allowClone": false, + "allowExport": false, + "enableAppTokens": false, + "hideFromPublic": false, + "mustBeRealmApproved": true, + "useIPFilter": true + }, + "timeZone": "(UTC-08:00) Pacific Time (US & Canada)", + "updated": "2020-04-03T19:12:20Z", + "dataClassification": "Confidential" +} \ No newline at end of file diff --git a/tests/fixtures/qb_api_responses/getAppTables.json b/tests/fixtures/qb_api_responses/getAppTables.json new file mode 100644 index 0000000..c721d94 --- /dev/null +++ b/tests/fixtures/qb_api_responses/getAppTables.json @@ -0,0 +1,38 @@ +[ + { + "name": "Example Table #0", + "created": "2020-03-27T18:34:40Z", + "updated": "2020-04-03T19:12:40Z", + "alias": "_dbid_example_table_#0", + "description": "Table zero as an example.", + "id": "bpqe82s1", + "nextRecordId": 1, + "nextFieldId": 1, + "defaultSortFieldId": 2, + "defaultSortOrder": "ASC", + "keyFieldId": 3, + "singleRecordName": "Example Record", + "pluralRecordName": "Example Records", + "sizeLimit": "150 MB", + "spaceUsed": "17 MB", + "spaceRemaining": "133 MB" + }, + { + "name": "Example Table #2", + "created": "2020-03-27T18:34:40Z", + "updated": "2020-04-03T19:12:40Z", + "alias": "_dbid_example_table_#2", + "description": "Table two as an example.", + "id": "bpqe82z1", + "nextRecordId": 10, + "nextFieldId": 111, + "defaultSortFieldId": 12, + "defaultSortOrder": "DESC", + "keyFieldId": 33, + "singleRecordName": "Another Record", + "pluralRecordName": "More Records", + "sizeLimit": "150 MB", + "spaceUsed": "22 MB", + "spaceRemaining": "128 MB" + } +] \ No newline at end of file diff --git a/tests/fixtures/qb_api_responses/getFields.json b/tests/fixtures/qb_api_responses/getFields.json new file mode 100644 index 0000000..20343d2 --- /dev/null +++ b/tests/fixtures/qb_api_responses/getFields.json @@ -0,0 +1,81 @@ +[ + { + "id": 1, + "label": "Field1", + "fieldType": "text", + "noWrap": false, + "bold": false, + "required": false, + "appearsByDefault": false, + "findEnabled": false, + "unique": false, + "doesDataCopy": false, + "fieldHelp": "field help", + "audited": false, + "properties": { + "primaryKey": false, + "foreignKey": false, + "numLines": 1, + "maxLength": 0, + "appendOnly": false, + "allowHTML": false, + "sortAsGiven": false, + "carryChoices": true, + "allowNewChoices": false, + "formula": "", + "defaultValue": "" + }, + "permissions": [ + { + "permissionType": "View", + "role": "Viewer", + "roleId": 10 + }, + { + "permissionType": "None", + "role": "Participant", + "roleId": 11 + }, + { + "permissionType": "Modify", + "role": "Administrator", + "roleId": 12 + } + ] + }, + { + "id": 2, + "label": "Numeric Field", + "fieldType": "numeric", + "appearsByDefault": true, + "audited": false, + "bold": false, + "fieldHelp": "This is a numeric field", + "findEnabled": false, + "addToForms": true, + "properties": { + "numberFormat": 3, + "decimalPlaces": 2, + "doesAverage": true, + "doesTotal": true, + "blankIsZero": true + }, + "permissions": [ + { + "role": "Viewer", + "permissionType": "View", + "roleId": 10 + }, + { + "role": "Participant", + "permissionType": "Modify", + "roleId": 11 + }, + { + "role": "Administrator", + "permissionType": "Modify", + "roleId": 12 + } + ] + } +] \ No newline at end of file diff --git a/tests/fixtures/qb_api_responses/upsert.json b/tests/fixtures/qb_api_responses/upsert.json new file mode 100644 index 0000000..d227262 --- /dev/null +++ b/tests/fixtures/qb_api_responses/upsert.json @@ -0,0 +1,39 @@ +{ + "data": [ + { + "1": { + "value": "Green" + }, + "2": { + "value": 42 + } + }, + { + "1": { + "value": "Red" + }, + "2": { + "value": 101 + } + }, + { + "1": { + "value": "Blue" + }, + "2": { + "value": 999 + } + } + ], + "metadata": { + "createdRecordIds": [ + 11, + 12 + ], + "totalNumberOfRecordsProcessed": 3, + "unchangedRecordIds": [], + "updatedRecordIds": [ + 1 + ] + } +} \ No newline at end of file diff --git a/tests/test_qbclient_client.py b/tests/test_qbclient_client.py new file mode 100644 index 0000000..06e5e4b --- /dev/null +++ b/tests/test_qbclient_client.py @@ -0,0 +1,100 @@ +# ruff: noqa: N803, PLR2004, PD901, SLF001 + +import os +from unittest import mock + +import pandas as pd +import pytest + +from hrqb.exceptions import QBFieldNotFoundError + + +def test_qbclient_init(qbclient): + assert ( + qbclient.request_headers["Authorization"] + == f"QB-USER-TOKEN {os.environ['QUICKBASE_API_TOKEN']}" + ) + assert qbclient.app_id == os.environ["QUICKBASE_APP_ID"] + + +def test_qbclient_cache_api_request_response(qbclient, mocked_qb_api_getApp): + assert qbclient._cache == {} + first_response = qbclient.get_app_info() + with mock.patch("requests.get") as mocked_requests_get: + mocked_requests_get.return_value = mocked_qb_api_getApp + second_response = qbclient.get_app_info() + assert first_response == second_response + assert ("apps/qb-app-def456", "{}") in qbclient._cache + mocked_requests_get.assert_not_called() + + +def test_qbclient_get_app_info(qbclient, mocked_qb_api_getApp): + assert qbclient.get_app_info() == mocked_qb_api_getApp + + +def test_qbclient_get_tables(qbclient, mocked_qb_api_getAppTables): + assert len(mocked_qb_api_getAppTables) == 2 + assert mocked_qb_api_getAppTables[0]["name"] == "Example Table #0" + df = pd.DataFrame(mocked_qb_api_getAppTables) + assert df.equals(qbclient.get_tables()) + + +def test_qbclient_get_table_id(qbclient, mocked_qb_api_getAppTables, mocked_table_id): + assert qbclient.get_table_id("Example Table #0") == mocked_table_id + + +def test_qbclient_get_table_fields(qbclient, mocked_qb_api_getFields, mocked_table_id): + assert len(mocked_qb_api_getFields) == 2 + assert mocked_qb_api_getFields[0]["label"] == "Field1" + df = pd.DataFrame(mocked_qb_api_getFields) + assert df.equals(qbclient.get_table_fields(mocked_table_id)) + + +def test_qbclient_get_table_fields_name_to_id( + qbclient, + mocked_table_id, + mocked_qb_api_getFields, +): + field_name_to_ids = qbclient.get_table_fields_name_to_id(mocked_table_id) + assert field_name_to_ids["Field1"] == 1 + assert field_name_to_ids["Numeric Field"] == 2 + + +def test_qbclient_prepare_upsert_payload( + qbclient, mocked_table_id, mocked_upsert_data, mocked_upsert_payload +): + upsert_payload = qbclient.prepare_upsert_payload( + mocked_table_id, mocked_upsert_data, None + ) + assert upsert_payload == mocked_upsert_payload + + +def test_qbclient_prepare_upsert_payload_sets_merge_field( + qbclient, mocked_table_id, mocked_upsert_data +): + merge_field_name, merge_field_id = "Numeric Field", 2 + upsert_payload = qbclient.prepare_upsert_payload( + mocked_table_id, mocked_upsert_data, merge_field=merge_field_name + ) + assert upsert_payload["mergeFieldId"] == merge_field_id + + +def test_qbclient_upsert_records_bad_field_name_error( + qbclient, mocked_table_id, mocked_qb_api_getFields +): + records = [ + {"Field1": "Green", "Numeric Field": 42}, + {"Field1": "Red", "Numeric Field": 101}, + {"Field1": "Blue", "Numeric Field": 999}, + {"Field1": "Blue", "Numeric Field": 999, "IAmBadField": "I will error."}, + ] + with pytest.raises(QBFieldNotFoundError, match="Field name 'IAmBadField' not found"): + qbclient.prepare_upsert_payload(mocked_table_id, records, merge_field=None) + + +def test_qbclient_upsert_records_success( + qbclient, + mocked_qb_api_upsert, + mocked_upsert_payload, +): + assert qbclient.upsert_records(mocked_upsert_payload) == mocked_qb_api_upsert