diff --git a/.nvmrc b/.nvmrc index 28c34d2c..9ddeebac 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -14.19.0 +14.19.1 diff --git a/.vscode/launch.json b/.vscode/launch.json index 2178f254..fb075937 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -48,17 +48,34 @@ "request": "launch", "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/jest", "args": [ - "-p ${input:pattern}", - // "--runInBand", + "-p http", + "--runInBand", // "--no-cache", - "--watch" + "--watch", + "-f" + ], + "cwd": "${workspaceRoot}", + "protocol": "inspector", + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen" + }, + { + "name": "Jest Debug tests watch mode specific file", + "type": "node", + "request": "launch", + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/jest", + "args": [ + "src/integration/http.test.ts", + "--runInBand", + // "--no-cache", + "--watch", + "-f" ], "cwd": "${workspaceRoot}", "protocol": "inspector", "console": "integratedTerminal", "internalConsoleOptions": "neverOpen" }, - { "name": "Inpyjamas scripts Debug tests watch mode", "type": "node", diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ab66ee4..a8e261bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,29 +1,44 @@ -# [3.1.0](https://github.com/technologiestiftung/stadtpuls-api/compare/v3.0.2...v3.1.0) (2022-03-30) +# [3.2.0-rc.2](https://github.com/technologiestiftung/stadtpuls-api/compare/v3.2.0-rc.1...v3.2.0-rc.2) (2022-05-03) -### Features +### Bug Fixes -* **error messages:** Add custom error response ([068eb1f](https://github.com/technologiestiftung/stadtpuls-api/commit/068eb1faf9178c6ae272849e0061034dfdd27fa2)) -* **http integration:** allow additional props in body ([9487983](https://github.com/technologiestiftung/stadtpuls-api/commit/9487983046429e7b8e6404914b29585de68c6cdc)) -* **http:** adds ability to post with own recorded_at prop ([83966c2](https://github.com/technologiestiftung/stadtpuls-api/commit/83966c2e17e6e4c5c12f2aa4d2e15ce1f46ba7a7)) +* **HTTP:** Remove max number of records ([f996c56](https://github.com/technologiestiftung/stadtpuls-api/commit/f996c5601334f494a905478754dd0c3dbc03e338)) -# [3.1.0-rc.2](https://github.com/technologiestiftung/stadtpuls-api/compare/v3.1.0-rc.1...v3.1.0-rc.2) (2022-03-30) +# [3.2.0-rc.1](https://github.com/technologiestiftung/stadtpuls-api/compare/v3.1.0...v3.2.0-rc.1) (2022-05-03) + + +### Bug Fixes + +* **deps:** update react monorepo to v18 ([7caef30](https://github.com/technologiestiftung/stadtpuls-api/commit/7caef30be2873989a33a10361e5f4f86e70e32c6)) +* **http:** fixes failing logic and adds tests ([e030419](https://github.com/technologiestiftung/stadtpuls-api/commit/e03041984368fedb6a16649fc42060457b2d7efa)) ### Features -* **error messages:** Add custom error response ([068eb1f](https://github.com/technologiestiftung/stadtpuls-api/commit/068eb1faf9178c6ae272849e0061034dfdd27fa2)) -* **http:** adds ability to post with own recorded_at prop ([83966c2](https://github.com/technologiestiftung/stadtpuls-api/commit/83966c2e17e6e4c5c12f2aa4d2e15ce1f46ba7a7)) +* **http:** adds batch posting of records ([510d433](https://github.com/technologiestiftung/stadtpuls-api/commit/510d433c358f50360890918d583709cdfa2cf1a9)) -# [3.1.0-rc.1](https://github.com/technologiestiftung/stadtpuls-api/compare/v3.0.2...v3.1.0-rc.1) (2022-03-29) +# [3.1.0](https://github.com/technologiestiftung/stadtpuls-api/compare/v3.0.2...v3.1.0) (2022-03-30) + + +### Bug Fixes + +* **http:** fixes failing logic and adds tests ([e030419](https://github.com/technologiestiftung/stadtpuls-api/commit/e03041984368fedb6a16649fc42060457b2d7efa)) +* **deps:** update react monorepo to v18 ([7caef30](https://github.com/technologiestiftung/stadtpuls-api/commit/7caef30be2873989a33a10361e5f4f86e70e32c6)) ### Features +* **http:** adds batch posting of records ([510d433](https://github.com/technologiestiftung/stadtpuls-api/commit/510d433c358f50360890918d583709cdfa2cf1a9)) +* **error messages:** Add custom error response ([068eb1f](https://github.com/technologiestiftung/stadtpuls-api/commit/068eb1faf9178c6ae272849e0061034dfdd27fa2)) +* **http integration:** allow additional props in body ([9487983](https://github.com/technologiestiftung/stadtpuls-api/commit/9487983046429e7b8e6404914b29585de68c6cdc)) +* **http:** adds ability to post with own recorded_at prop ([83966c2](https://github.com/technologiestiftung/stadtpuls-api/commit/83966c2e17e6e4c5c12f2aa4d2e15ce1f46ba7a7)) +* **error messages:** Add custom error response ([068eb1f](https://github.com/technologiestiftung/stadtpuls-api/commit/068eb1faf9178c6ae272849e0061034dfdd27fa2)) +* **http:** adds ability to post with own recorded_at prop ([83966c2](https://github.com/technologiestiftung/stadtpuls-api/commit/83966c2e17e6e4c5c12f2aa4d2e15ce1f46ba7a7)) * **http integration:** allow additional props in body ([9487983](https://github.com/technologiestiftung/stadtpuls-api/commit/9487983046429e7b8e6404914b29585de68c6cdc)) -## [3.0.2](https://github.com/technologiestiftung/stadtpuls-api/compare/v3.0.1...v3.0.2) (2022-03-23) +## [3.0.2](https://github.com/technologiestiftung/stadtpuls-api/compare/v3.0.1...v3.0.2) (2022-03-23) ### Bug Fixes diff --git a/dev-tools/dev-client/package-lock.json b/dev-tools/dev-client/package-lock.json index 8bc48010..f09e1795 100644 --- a/dev-tools/dev-client/package-lock.json +++ b/dev-tools/dev-client/package-lock.json @@ -2701,7 +2701,8 @@ "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", + "dev": true }, "object-hash": { "version": "2.2.0", @@ -3126,22 +3127,20 @@ "dev": true }, "react": { - "version": "17.0.2", - "resolved": "https://registry.npmjs.org/react/-/react-17.0.2.tgz", - "integrity": "sha512-gnhPt75i/dq/z3/6q/0asP78D0u592D5L1pd7M8P+dck6Fu/jJeL6iVVK23fptSUZj8Vjf++7wXA8UNclGQcbA==", + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/react/-/react-18.0.0.tgz", + "integrity": "sha512-x+VL6wbT4JRVPm7EGxXhZ8w8LTROaxPXOqhlGyVSrv0sB1jkyFGgXxJ8LVoPRLvPR6/CIZGFmfzqUa2NYeMr2A==", "requires": { - "loose-envify": "^1.1.0", - "object-assign": "^4.1.1" + "loose-envify": "^1.1.0" } }, "react-dom": { - "version": "17.0.2", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-17.0.2.tgz", - "integrity": "sha512-s4h96KtLDUQlsENhMn1ar8t2bEa+q/YAtj8pPPdIjPDGBDIVNsrD9aXNWqspUe6AzKCIG0C1HZZLqLV7qpOBGA==", + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.0.0.tgz", + "integrity": "sha512-XqX7uzmFo0pUceWFCt7Gff6IyIMzFUn7QMZrbrQfGxtaxXZIcGQzoNpRLE3fQLnS4XzLLPMZX2T9TRcSrasicw==", "requires": { "loose-envify": "^1.1.0", - "object-assign": "^4.1.1", - "scheduler": "^0.20.2" + "scheduler": "^0.21.0" } }, "react-is": { @@ -3334,12 +3333,11 @@ "dev": true }, "scheduler": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.2.tgz", - "integrity": "sha512-2eWfGgAqqWFGqtdMmcL5zCMK1U8KlXv8SQFGglL3CEtd0aDVDWgeF/YoCmvln55m5zSk3J/20hTaSBeSObsQDQ==", + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.21.0.tgz", + "integrity": "sha512-1r87x5fz9MXqswA2ERLo0EbOAU74DpIUO090gIasYTqlVoJeMcl+Z1Rg7WHz+qtPujhS/hGIt9kxZOYBV3faRQ==", "requires": { - "loose-envify": "^1.1.0", - "object-assign": "^4.1.1" + "loose-envify": "^1.1.0" } }, "semver": { diff --git a/dev-tools/dev-client/package.json b/dev-tools/dev-client/package.json index 334aa0cf..5afcfe5b 100644 --- a/dev-tools/dev-client/package.json +++ b/dev-tools/dev-client/package.json @@ -10,8 +10,8 @@ }, "dependencies": { "@supabase/supabase-js": "1.21.3", - "react": "17.0.2", - "react-dom": "17.0.2" + "react": "18.0.0", + "react-dom": "18.0.0" }, "devDependencies": { "@snowpack/plugin-dotenv": "2.1.0", diff --git a/dev-tools/k6/docker-compose.yml b/dev-tools/k6/docker-compose.yml index 7504f68f..bde00d06 100644 --- a/dev-tools/k6/docker-compose.yml +++ b/dev-tools/k6/docker-compose.yml @@ -1,7 +1,7 @@ version: "3" services: k6: - image: loadimpact/k6:0.36.0 + image: loadimpact/k6:0.37.0 volumes: - ./scripts:/scripts command: ["run", "/scripts/memory-leaks.js"] diff --git a/dev-tools/rest-client/api.http b/dev-tools/rest-client/api.http index 0148d9b4..4e041483 100644 --- a/dev-tools/rest-client/api.http +++ b/dev-tools/rest-client/api.http @@ -21,7 +21,7 @@ @service_role_key = {{ $dotenv %SUPABASE_SERVICE_ROLE_KEY }} @auth_token = {{ $dotenv %AUTH_TOKEN }} -@sensor_id = 3 +@sensor_id = {{ $dotenv %SENSOR_ID }} @nice_id = {{ $dotenv %NICE_ID }} @@ -119,6 +119,18 @@ Prefer: return=representation # } # ] +### delete a record in database using postgrest + +# ▄▄▄ ██▓███ ██▓ +# ▒████▄ ▓██░ ██▒▓██▒ +# ▒██ ▀█▄ ▓██░ ██▓▒▒██▒ +# ░██▄▄▄▄██ ▒██▄█▓▒ ▒░██░ +# ▓█ ▓██▒▒██▒ ░ ░░██░ +# ▒▒ ▓▒█░▒▓▒░ ░ ░░▓ +# ▒ ▒▒ ░░▒ ░ ▒ ░ +# ░ ▒ ░░ ▒ ░ +# ░ ░ ░ + ### healthcheck @@ -216,4 +228,37 @@ POST {{ baseurl }}/integrations/ttn/v3 Content-Type: application/json Authorization: Bearer {{auth_token}} -{"simulated":true,"end_device_ids":{"application_ids":{"application_id":"foo"},"device_id":"foo"},"received_at":"2021-10-04T14:36:58.082Z","uplink_message":{"decoded_payload":{"foo":"bah","measurements":[1,2,3],"bytes":[1,2,3]},"locations":{"user":{"latitude":13,"longitude":52,"altitude":23,"source":"SOURCE_REGISTRY"}}}} \ No newline at end of file +{"simulated":true,"end_device_ids":{"application_ids":{"application_id":"foo"},"device_id":"foo"},"received_at":"2021-10-04T14:36:58.082Z","uplink_message":{"decoded_payload":{"foo":"bah","measurements":[1,2,3],"bytes":[1,2,3]},"locations":{"user":{"latitude":13,"longitude":52,"altitude":23,"source":"SOURCE_REGISTRY"}}}} + + +### records create + + +POST {{supabase_url}}/rest/v1/records +Content-Type: application/json +apikey: {{anon_key}} +Authorization: Bearer {{user_token}} +Prefer: return=representation + +{"measurements":[1], "recorded_at":"2021-10-04T14:36:58.082Z","sensor_id": {{sensor_id}}} + + +### update a record + +PATCH {{supabase_url}}/rest/v1/records?id=eq.2 +Content-Type: application/json +apikey: {{anon_key}} +Authorization: Bearer {{user_token}} +Prefer: return=representation + +{"measurements":[5,2,3,4], "recorded_at":"2021-10-04T14:36:58.082Z","sensor_id": {{sensor_id}}} + + + +### delete a record + +DELETE {{supabase_url}}/rest/v1/records?id=eq.2&sensor_id=eq.{{sensor_id}} +Content-Type: application/json +apikey: {{anon_key}} +Authorization: Bearer {{user_token}} +Prefer: return=representation diff --git a/package-lock.json b/package-lock.json index ee38631b..8b60da45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "@technologiestiftung/stadtpuls-api", - "version": "3.1.0", + "version": "3.2.0-rc.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -2694,9 +2694,9 @@ }, "dependencies": { "debug": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", - "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "requires": { "ms": "2.1.2" } @@ -4322,6 +4322,12 @@ "whatwg-url": "^8.0.0" } }, + "date-fns": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.28.0.tgz", + "integrity": "sha512-8d35hViGYx/QH0icHYCeLmsLmMUheMmTyV9Fcm6gvNwdw31yXXH+O85sOBJ+OLnLQMKZowvpKb6FgMIQjcpvQw==", + "dev": true + }, "dateformat": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-1.0.12.tgz", @@ -5678,9 +5684,9 @@ } }, "fastify": { - "version": "3.27.4", - "resolved": "https://registry.npmjs.org/fastify/-/fastify-3.27.4.tgz", - "integrity": "sha512-SOfnHBxG9zxCSIvt6aHoR/cao8QBddWmGP/mb5KQKRc+KI1kB7b79M2hCDOTSyHdLAF2OX+oI6X3weeLc+MqKg==", + "version": "3.28.0", + "resolved": "https://registry.npmjs.org/fastify/-/fastify-3.28.0.tgz", + "integrity": "sha512-LAQtGllpkRe8L6Tpf3zdbvXzXFOrgaWV3Tbvp3xMv9ngcr9zht9U2/mo5zq9qp9kplSiBJ0w43aVAMqv6PBMbw==", "requires": { "@fastify/ajv-compiler": "^1.0.0", "abstract-logging": "^2.0.0", @@ -5712,11 +5718,6 @@ "quick-format-unescaped": "^4.0.3", "sonic-boom": "^1.0.2" } - }, - "tiny-lru": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz", - "integrity": "sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg==" } } }, @@ -8097,9 +8098,9 @@ } }, "light-my-request": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-4.8.0.tgz", - "integrity": "sha512-C2XESrTRsZnI59NSQigOsS6IuTxpj8OhSBvZS9fhgBMsamBsAuWN1s4hj/nCi8EeZcyAA6xbROhsZy7wKdfckg==", + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-4.9.0.tgz", + "integrity": "sha512-b1U3z4OVPoO/KanT14NRkXMr9rRtXAiq0ORqNrqhDyb5bGkZjAdEc6GRN1GWCfgaLBG+aq73qkCLDNeB3c2sLw==", "requires": { "ajv": "^8.1.0", "cookie": "^0.4.0", @@ -8108,9 +8109,9 @@ }, "dependencies": { "ajv": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.10.0.tgz", - "integrity": "sha512-bzqAEZOjkrUMl2afH8dknrq5KEk2SrwdBROR+vH1EKVQTqaUbJVPdc/gEdggTMM0Se+s+Ja4ju4TlNcStKl2Hw==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", + "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", "requires": { "fast-deep-equal": "^3.1.1", "json-schema-traverse": "^1.0.0", @@ -11641,10 +11642,11 @@ } }, "pino": { - "version": "7.9.0", - "resolved": "https://registry.npmjs.org/pino/-/pino-7.9.0.tgz", - "integrity": "sha512-5Zjb92wQPeCmTpIkdEjtz0LElnsAhL3VIgXm9yj3rr0+i7HcsxeiO46bA73sd9oys19qldR54mXURplUdMTf2w==", + "version": "7.10.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-7.10.0.tgz", + "integrity": "sha512-T6R92jy/APDElEuOk0gqa4nds3ZgqFbHde2X0g8XorlyPlVGlr9T5KQphtp72a3ByKOdZMg/gM/0IprpGQfTWg==", "requires": { + "atomic-sleep": "^1.0.0", "fast-redact": "^3.0.0", "on-exit-leak-free": "^0.2.0", "pino-abstract-transport": "v0.5.0", @@ -11654,35 +11656,21 @@ "real-require": "^0.1.0", "safe-stable-stringify": "^2.1.0", "sonic-boom": "^2.2.1", - "thread-stream": "^0.13.0" + "thread-stream": "^0.15.1" }, "dependencies": { - "pino-abstract-transport": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-0.5.0.tgz", - "integrity": "sha512-+KAgmVeqXYbTtU2FScx1XS3kNyfZ5TrXY07V96QnUSFqo2gAqlvmaxH67Lj7SWazqsMabf+58ctdTcBgnOLUOQ==", - "requires": { - "duplexify": "^4.1.2", - "split2": "^4.0.0" - } - }, "pino-std-serializers": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-4.0.0.tgz", "integrity": "sha512-cK0pekc1Kjy5w9V2/n+8MkZwusa6EyyxfeQCB799CQRhRt/CqYKiWs5adeu8Shve2ZNffvfC/7J64A2PJo1W/Q==" }, "sonic-boom": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-2.6.0.tgz", - "integrity": "sha512-6xYZFRmDEtxGqfOKcDQ4cPLrNa0SPEDI+wlzDAHowXE6YV42NeXqg9mP2KkiM8JVu3lHfZ2iQKYlGOz+kTpphg==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-2.7.0.tgz", + "integrity": "sha512-Ynxp0OGQG91wvDjCbFlRMHbSUmDq7dE/EgDeUJ/j+Q9x1FVkFry20cjLykxRSmlm3QS0B4JYAKE8239XKN4SHQ==", "requires": { "atomic-sleep": "^1.0.0" } - }, - "split2": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz", - "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==" } } }, @@ -11751,9 +11739,9 @@ } }, "pino-pretty": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-7.5.4.tgz", - "integrity": "sha512-p5AuDpsesgCiEE+8veQTOcuHBdj/BsAMntpDQeTJ0vAcsIGufR3WnKAXbjaELfEp1UYBD+ewAPb1EmIcfVXl6w==", + "version": "7.6.1", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-7.6.1.tgz", + "integrity": "sha512-H7N6ZYkiyrfwBGW9CSjx0uyO9Q2Lyt73881+OTYk8v3TiTdgN92QHrWlEq/LeWw5XtDP64jeSk3mnc6T+xX9/w==", "requires": { "args": "^5.0.1", "colorette": "^2.0.7", @@ -11781,9 +11769,9 @@ "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==" }, "sonic-boom": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-2.6.0.tgz", - "integrity": "sha512-6xYZFRmDEtxGqfOKcDQ4cPLrNa0SPEDI+wlzDAHowXE6YV42NeXqg9mP2KkiM8JVu3lHfZ2iQKYlGOz+kTpphg==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-2.7.0.tgz", + "integrity": "sha512-Ynxp0OGQG91wvDjCbFlRMHbSUmDq7dE/EgDeUJ/j+Q9x1FVkFry20cjLykxRSmlm3QS0B4JYAKE8239XKN4SHQ==", "requires": { "atomic-sleep": "^1.0.0" } @@ -13718,9 +13706,9 @@ "dev": true }, "thread-stream": { - "version": "0.13.2", - "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-0.13.2.tgz", - "integrity": "sha512-woZFt0cLFkPdhsa+IGpRo1jiSouaHxMIljzTgt30CMjBWoUYbbcHqnunW5Yv+BXko9H05MVIcxMipI3Jblallw==", + "version": "0.15.1", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-0.15.1.tgz", + "integrity": "sha512-SCnuIT27gc2h/F/RY2peuC7brgLy+1oXU+7yOIAITz1z5stDpXCF5rAoFcykjuK6ifbTlKAHL7Ccq8oc5Btv1w==", "requires": { "real-require": "^0.1.0" } diff --git a/package.json b/package.json index 47d109fd..7ea9484b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@technologiestiftung/stadtpuls-api", "private": true, - "version": "3.1.0", + "version": "3.2.0-rc.2", "description": "An API for the stadtpuls.com project", "author": "ff6347 ", "homepage": "https://github.com/technologiestiftung/stadtpuls-api/tree/main/packages/stadtpuls-api#readme", @@ -58,6 +58,7 @@ "dotenv": "10.0.0", "esbuild": "0.12.28", "eslint": "7.32.0", + "date-fns": "2.28.0", "glob": "7.2.0", "is-ci": "3.0.1", "jest-each": "27.5.1", @@ -76,7 +77,7 @@ "ajv-errors": "1.0.1", "bcrypt": "5.0.1", "config": "3.3.7", - "fastify": "3.27.4", + "fastify": "3.28.0", "fastify-auth": "1.1.0", "fastify-blipp": "3.1.0", "fastify-cors": "6.0.3", @@ -90,9 +91,9 @@ "ioredis": "4.28.5", "make-promises-safe": "5.1.0", "pg": "8.7.3", - "pino": "7.9.0", + "pino": "7.10.0", "pino-logflare": "0.3.12", - "pino-pretty": "7.5.4", + "pino-pretty": "7.6.1", "pino-syslog": "2.0.0", "uuid": "8.3.2" }, diff --git a/src/__test-utils/create-records.ts b/src/__test-utils/create-records.ts index 3c4c8147..07254f61 100644 --- a/src/__test-utils/create-records.ts +++ b/src/__test-utils/create-records.ts @@ -2,7 +2,7 @@ // // This software is released under the MIT License. // https://opensource.org/licenses/MIT - +import { sub } from "date-fns"; import { supabase } from "./index"; import { definitions } from "@technologiestiftung/stadtpuls-supabase-definitions"; // const binomial = require("@stdlib/random/base/binomial"); @@ -47,3 +47,39 @@ export async function createRecords({ } return data; } + +export async function createRecordsPayload({ + amount, +}: { + amount: number; +}): Promise< + { + recorded_at: string; + measurements: number[]; + latitude?: number; + longitude?: number; + altitude?: number; + }[] +> { + if (amount <= 0) throw new Error("amount must not be negative"); + const records = []; + const now = new Date(); + for (let i = 0; i < amount; i++) { + const measurements = [Math.random() * 10]; + const recorded_at = sub(now.setUTCHours(i % 24), { + minutes: i, + }).toISOString(); + const latitude = Math.random() * 10; + const longitude = Math.random() * 10; + const altitude = Math.random() * 10; + records.push({ + recorded_at, + measurements, + latitude, + longitude, + altitude, + }); + } + + return records; +} diff --git a/src/integrations/http.batch-insert.test.ts b/src/integrations/http.batch-insert.test.ts new file mode 100644 index 00000000..0067d94d --- /dev/null +++ b/src/integrations/http.batch-insert.test.ts @@ -0,0 +1,303 @@ +/* eslint-disable jest/require-top-level-describe */ +/* eslint-disable jest/no-hooks */ +import buildServer from "../lib/server"; +import each from "jest-each"; +import { + deleteUser, + jwtSecret, + supabaseServiceRoleKey, + supabaseUrl, + apiVersion, + signupUser, + createSensor, + truncateTables, + closePool, + connectPool, +} from "../__test-utils"; +import { createAuthToken } from "../__test-utils/create-auth-token"; +import { createRecordsPayload } from "../__test-utils/create-records"; + +const issuer = "tsb"; +const buildServerOpts = { + jwtSecret, + supabaseUrl, + supabaseServiceRoleKey, + logger: false, + issuer, +}; + +const httpPayload = { + latitude: 52.483107, + longitude: 13.390679, + altitude: 30, + measurements: [1, 2, 3], + recorded_at: "2022-03-30T00:00:00.000Z", +}; + +describe("tests for the http integration", () => { + // eslint-disable-next-line jest/no-hooks + beforeAll(async () => { + await connectPool(); + }); + beforeEach(async () => { + await truncateTables(); + }); + // eslint-disable-next-line jest/no-hooks + afterAll(async () => { + await truncateTables(); + await closePool(); + }); + + test("should allow array of data or object in body", async () => { + // start boilerplate setup test + const server = buildServer(buildServerOpts); + const user = await signupUser(); + + const authToken = await createAuthToken({ + server, + userToken: user.token, + }); + const sensor = await createSensor({ + user_id: user.id, + }); + // end boilerplate + const records = await createRecordsPayload({ + amount: 100, + }); + const responseArray = await server.inject({ + method: "POST", + url: `/api/v${apiVersion}/sensors/${sensor.id}/records`, + payload: { records }, + headers: { + Authorization: `Bearer ${authToken}`, + }, + }); + const responseObject = await server.inject({ + method: "POST", + url: `/api/v${apiVersion}/sensors/${sensor.id}/records`, + payload: records[0], + headers: { + Authorization: `Bearer ${authToken}`, + }, + }); + expect(responseArray.statusCode).toBe(201); + expect(responseObject.statusCode).toBe(201); + // start boilerplate delete user + await deleteUser(user.token); + // end boilerplate + }); + + test("should prefer array of data before object in body", async () => { + // start boilerplate setup test + const server = buildServer(buildServerOpts); + const user = await signupUser(); + + const authToken = await createAuthToken({ + server, + userToken: user.token, + }); + const sensor = await createSensor({ + user_id: user.id, + }); + // end boilerplate + const records = await createRecordsPayload({ + amount: 2, + }); + const responseArray = await server.inject({ + method: "POST", + url: `/api/v${apiVersion}/sensors/${sensor.id}/records`, + payload: { records, measurements: [1, 2, 3] }, + headers: { + Authorization: `Bearer ${authToken}`, + }, + }); + expect(responseArray.statusCode).toBe(201); + expect(responseArray.json().records).toBeDefined(); + expect(responseArray.json().record).not.toBeDefined(); + expect(responseArray.json().records.length).toBeGreaterThan(1); + // start boilerplate delete user + await deleteUser(user.token); + // end boilerplate + }); + + test(`should allow around 1 mb of payload`, async () => { + // start boilerplate setup test + const server = buildServer(buildServerOpts); + const user = await signupUser(); + + const authToken = await createAuthToken({ + server, + userToken: user.token, + }); + const sensor = await createSensor({ + user_id: user.id, + }); + // end boilerplate + const records = await createRecordsPayload({ + amount: 1, + }); + const oneMBInBytes = 1048576; // 1024 * 1024 + + let jsonString = JSON.stringify(records); + const bytesOfSkeleton = Buffer.byteLength(jsonString, "utf8"); + const bytesOfMeasurements = oneMBInBytes - bytesOfSkeleton; + + const measurements = []; + for (let i = 0; i < bytesOfMeasurements / 2 - 10; i++) { + measurements.push(1); + } + + records[0].measurements = measurements; + jsonString = JSON.stringify(records); + // console.log(bytesOfSkeleton, Buffer.byteLength(jsonString, "utf8")); + + const responseArray = await server.inject({ + method: "POST", + url: `/api/v${apiVersion}/sensors/${sensor.id}/records`, + payload: { records }, + headers: { + Authorization: `Bearer ${authToken}`, + }, + }); + expect(responseArray.statusCode).toBe(201); + + // start boilerplate delete user + await deleteUser(user.token); + // end boilerplate + }); + + test(`should reject large payload with 413`, async () => { + // start boilerplate setup test + const server = buildServer(buildServerOpts); + const user = await signupUser(); + + const authToken = await createAuthToken({ + server, + userToken: user.token, + }); + const sensor = await createSensor({ + user_id: user.id, + }); + // end boilerplate + const records = await createRecordsPayload({ + amount: 1, + }); + const oneMBInBytes = 1048576; // 1024 * 1024 + + let jsonString = JSON.stringify(records); + const bytesOfSkeleton = Buffer.byteLength(jsonString, "utf8"); + const bytesOfMeasurements = oneMBInBytes - bytesOfSkeleton; + + const measurements = []; + for (let i = 0; i < bytesOfMeasurements; i++) { + measurements.push(1); + } + + records[0].measurements = measurements; + jsonString = JSON.stringify(records); + // console.log(bytesOfSkeleton, Buffer.byteLength(jsonString, "utf8")); + + const responseArray = await server.inject({ + method: "POST", + url: `/api/v${apiVersion}/sensors/${sensor.id}/records`, + payload: { records }, + headers: { + Authorization: `Bearer ${authToken}`, + }, + }); + expect(responseArray.statusCode).toBe(413); + expect(responseArray.json()).toMatchInlineSnapshot(` + Object { + "code": "FST_ERR_CTP_BODY_TOO_LARGE", + "error": "Payload Too Large", + "message": "Request body is too large", + "statusCode": 413, + } + `); + // start boilerplate delete user + await deleteUser(user.token); + // end boilerplate + }); + test(`should reject records missing measurements`, async () => { + // start boilerplate setup test + const server = buildServer(buildServerOpts); + const user = await signupUser(); + + const authToken = await createAuthToken({ + server, + userToken: user.token, + }); + const sensor = await createSensor({ + user_id: user.id, + }); + // end boilerplate + const records = await createRecordsPayload({ + amount: 1000, + }); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + records[5].measurements = undefined; + const responseArray = await server.inject({ + method: "POST", + url: `/api/v${apiVersion}/sensors/${sensor.id}/records`, + payload: { records }, + headers: { + Authorization: `Bearer ${authToken}`, + }, + }); + expect(responseArray.statusCode).toBe(400); + expect(responseArray.json()).toMatchInlineSnapshot(` + Object { + "error": "Bad Request", + "message": "body/records/5 should have required property 'measurements', body should match \\"then\\" schema", + "statusCode": 400, + } + `); + // start boilerplate delete user + await deleteUser(user.token); + // end boilerplate + }); + + test(`should reject records missing recorded_at`, async () => { + // start boilerplate setup test + const server = buildServer(buildServerOpts); + const user = await signupUser(); + + const authToken = await createAuthToken({ + server, + userToken: user.token, + }); + const sensor = await createSensor({ + user_id: user.id, + }); + // end boilerplate + const records = await createRecordsPayload({ + amount: 1000, + }); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + records[5].recorded_at = undefined; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + records[100].recorded_at = undefined; + const responseArray = await server.inject({ + method: "POST", + url: `/api/v${apiVersion}/sensors/${sensor.id}/records`, + payload: { records }, + headers: { + Authorization: `Bearer ${authToken}`, + }, + }); + expect(responseArray.statusCode).toBe(400); + expect(responseArray.json()).toMatchInlineSnapshot(` + Object { + "error": "Bad Request", + "message": "body/records/5 should have required property 'recorded_at', body/records/100 should have required property 'recorded_at', body should match \\"then\\" schema", + "statusCode": 400, + } + `); + // start boilerplate delete user + await deleteUser(user.token); + // end boilerplate + }); +}); diff --git a/src/integrations/http.test.ts b/src/integrations/http.test.ts index 19a59660..bf7cce38 100644 --- a/src/integrations/http.test.ts +++ b/src/integrations/http.test.ts @@ -75,15 +75,30 @@ describe("tests for the http integration", () => { test("should be rejected due to no wrong body", async () => { const server = buildServer(buildServerOpts); + const user = await signupUser(); + + const authToken = await createAuthToken({ + server, + userToken: user.token, + }); + const sensor = await createSensor({ + user_id: user.id, + }); const response = await server.inject({ method: "POST", - url: `/api/v${apiVersion}/sensors/1/records`, + url: `/api/v${apiVersion}/sensors/${sensor.id}/records`, payload: {}, + headers: { + Authorization: `Bearer ${authToken}`, + }, }); expect(response.statusCode).toBe(400); expect(response.body).toMatchInlineSnapshot( `"{\\"statusCode\\":400,\\"error\\":\\"Bad Request\\",\\"message\\":\\"body should have required property 'measurements'\\"}"` ); + // start boilerplate + await deleteUser(user.token); + // end boilerplate }); test("should be rejected due to no authorization header", async () => { @@ -314,7 +329,7 @@ describe("tests for the http integration", () => { user_id: user.id, }); // end boilerplate - const recorded_at = "2020-01-01"; + const recorded_at = "abc"; const response = await server.inject({ method: "POST", url: `/api/v${apiVersion}/sensors/${device.id}/records`, @@ -330,7 +345,7 @@ describe("tests for the http integration", () => { expect(response.json()).toMatchInlineSnapshot(` Object { "error": "Bad Request", - "message": "body/recorded_at should match format \\"date-time\\" in ISO 8601 notation with UTC offset. Should be YYYY-MM-DDTHH:mm:ssZ or YYYY-MM-DDTHH:mm:ss+HH:mm or YYYY-MM-DDTHH:mm:ss-HH:mm-HH:mm", + "message": "recorded_at should match format 'date-time' in ISO 8601 notation with UTC offset. Should be YYYY-MM-DDTHH:mm:ssZ or YYYY-MM-DDTHH:mm:ss+HH:mm or YYYY-MM-DDTHH:mm:ss-HH:mm-HH:mm", "statusCode": 400, } `); diff --git a/src/integrations/http.ts b/src/integrations/http.ts index e9fde747..97190354 100644 --- a/src/integrations/http.ts +++ b/src/integrations/http.ts @@ -6,20 +6,27 @@ import { AuthToken } from "../common/jwt"; import S from "fluent-json-schema"; import config from "config"; import { logLevel } from "../lib/env"; +import { isValidDate } from "../lib/date-utils"; declare module "fastify" { interface FastifyInstance { verifyJWT: (request: FastifyRequest, reply: FastifyReply) => Promise; } } +type Record = definitions["records"]; -interface HTTPPostBody { +interface RecordPayload { latitude?: number; longitude?: number; altitude?: number; - recorded_at?: string; measurements: number[]; + recorded_at?: string; } +// +//https://stackoverflow.com/a/69328045 +type WithRequired = T & { [P in K]-?: T[P] }; +type RecordPayloadWithRecordedAt = WithRequired; +type HTTPPostBody = RecordPayload | { records: RecordPayloadWithRecordedAt[] }; interface HTTPPostParams { sensorId: string; @@ -27,9 +34,8 @@ interface HTTPPostParams { const apiVersion = config.get("apiVersion"); const mountPoint = config.get("mountPoint"); -const postHTTPBodySchema = S.object() - .id("/integration/http") - .title("Validation for data coming in via HTTP") +const recordSchema = S.object() + .id("#record") .additionalProperties(true) .raw({ errorMessage: { @@ -45,6 +51,25 @@ const postHTTPBodySchema = S.object() .prop("recorded_at", S.string().format("date-time")) .prop("measurements", S.array().items(S.number()).required()); +const recordsSchema = S.object() + .additionalProperties(true) + .prop( + "records", + S.array().items( + S.object().required(["measurements", "recorded_at"]).extend(recordSchema) + ) + ); + +const postHTTPBodySchema = S.object() + .id("/integration/http") + .title("Validation for data coming in via HTTP") + .additionalProperties(true) + .ifThenElse( + S.object().prop("records", S.array()), + recordsSchema, + recordSchema + ); + const postHTTPParamsSchema = S.object() .id("/integration/http/params") .title("HTTP Params") @@ -75,6 +100,7 @@ const http: FastifyPluginAsync = async (fastify) => { logLevel, preHandler: fastify.auth([fastify.verifyJWT]), handler: async (request, reply) => { + let isBatchUpdate = false; // --------------------------------- // TODO: [STADTPULS-474] remove duplicate code on both integrations const decoded = (await request.jwtVerify()) as AuthToken; @@ -113,47 +139,71 @@ const http: FastifyPluginAsync = async (fastify) => { "sensor not found postgres error" ); } - - const measurements = request.body.measurements; - const latitude = request.body.latitude; - const longitude = request.body.longitude; - const altitude = request.body.altitude; - const recorded_at_string = request.body.recorded_at; - - const { - data: updatedSensors, - error: updateError, - } = await fastify.supabase - .from("sensors") - .update({ - latitude, - longitude, - altitude, - }) - .eq("id", sensors[0].id); - if (updateError) { - fastify.log.error(updateError, "Error while updating lat, lon, alt"); - } - fastify.log.info(updatedSensors, "updated lat, lon, alt"); - - let recordedAt: string | undefined; - if (recorded_at_string) { - recordedAt = new Date(recorded_at_string).toISOString(); + let records: Omit[] = []; + if ("records" in request.body) { + isBatchUpdate = true; + fastify.log.info("insert multiple records. Wont update lat lon"); + records = [ + ...request.body.records.map((record) => { + return { + measurements: record.measurements, + recorded_at: record.recorded_at, + sensor_id: sensors[0].id, + }; + }), + ]; } else { - recordedAt = new Date().toISOString(); - } - - const { data: record, error: recordError } = await fastify.supabase - .from("records") - .insert([ + if (!request.body.measurements) { + throw fastify.httpErrors.badRequest( + "body should have required property 'measurements'" + ); + } + const measurements = request.body.measurements; + const latitude = request.body.latitude; + const longitude = request.body.longitude; + const altitude = request.body.altitude; + const recorded_at_string = request.body.recorded_at; + let recorded_at: string | undefined; + if (recorded_at_string !== undefined) { + // const parsedDate = parseISO(recorded_at_string); + if (!isValidDate(new Date(recorded_at_string))) { + throw fastify.httpErrors.badRequest( + "recorded_at should match format 'date-time' in ISO 8601 notation with UTC offset. Should be YYYY-MM-DDTHH:mm:ssZ or YYYY-MM-DDTHH:mm:ss+HH:mm or YYYY-MM-DDTHH:mm:ss-HH:mm-HH:mm" + ); + } + recorded_at = new Date(recorded_at_string).toISOString(); + } else { + recorded_at = new Date().toISOString(); + } + records = [ { - measurements: measurements, - recorded_at: recordedAt, + measurements, + recorded_at, sensor_id: sensors[0].id, }, - ]); + ]; + const { + data: updatedSensors, + error: updateError, + } = await fastify.supabase + .from("sensors") + .update({ + latitude, + longitude, + altitude, + }) + .eq("id", sensors[0].id); + if (updateError) { + fastify.log.error(updateError, "Error while updating lat, lon, alt"); + } + fastify.log.info(updatedSensors, "updated lat, lon, alt"); + } + + const { data, error: recordError } = await fastify.supabase + .from("records") + .insert(records); - if (!record) { + if (!records) { throw fastify.httpErrors.internalServerError("could not create record"); } if (recordError) { @@ -162,7 +212,9 @@ const http: FastifyPluginAsync = async (fastify) => { ); } - reply.status(201).send({ record }); + reply + .status(201) + .send(isBatchUpdate ? { records: data } : { record: data }); }, }); }; diff --git a/src/lib/__tests__/date-utils.test.ts b/src/lib/__tests__/date-utils.test.ts new file mode 100644 index 00000000..dec55e4a --- /dev/null +++ b/src/lib/__tests__/date-utils.test.ts @@ -0,0 +1,22 @@ +/* eslint-disable jest/require-top-level-describe */ +import { isValidDate } from "../date-utils"; +import each from "jest-each"; + +each([ + ["2022-12-31T12:00:00z", true], + ["foo", false], + [1, true], + [100000, true], + [new Date(), true], + [[], false], + [{}, false], + [undefined, false], + [true, true], //<-- this is wired + [null, true], //<-- this also +]).describe("date utils tests", (input, expected) => { + test(`returns ${expected}`, async () => { + const actual = isValidDate(new Date(input)); + // console.log(new Date(input), actual); + expect(actual).toBe(expected); + }); +}); diff --git a/src/lib/date-utils.ts b/src/lib/date-utils.ts new file mode 100644 index 00000000..6fda0952 --- /dev/null +++ b/src/lib/date-utils.ts @@ -0,0 +1,6 @@ +//https://stackoverflow.com/questions/1353684/detecting-an-invalid-date-date-instance-in-javascript +export function isValidDate(d: unknown): boolean { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + return d instanceof Date && !isNaN(d); +} diff --git a/src/lib/server.ts b/src/lib/server.ts index 3e62ae72..ac939a0f 100644 --- a/src/lib/server.ts +++ b/src/lib/server.ts @@ -14,7 +14,7 @@ import fastifySensible from "fastify-sensible"; import fastifyAuth from "fastify-auth"; import fastifyRateLimit from "fastify-rate-limit"; import ajvError from "ajv-errors"; -// TODO: Add useful formats for validation once we are in fastify 4 +// TODO: [BA-70] Add useful formats for validation once we are in fastify 4 // import ajvFormats from "ajv-formats"; import fastifySupabase from "./supabase"; @@ -71,7 +71,7 @@ export const buildServer: (options: { logger, ignoreTrailingSlash: true, exposeHeadRoutes: true, - // TODO: Update ajvError to latests once we are in fastify 4 + // TODO: [BA-71] Update ajvError to latests once we are in fastify 4 ajv: { plugins: [ajvError /*,[ajvFormats, { formats: ["iso-date-time"] }]*/], customOptions: {