From e9c18eed5900eb07868d9a8b16bbecf9c532a34a Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Thu, 16 Nov 2023 19:57:26 -0600 Subject: [PATCH] feat: upgrade to response-rejection plugin --- .changeset/empty-crews-attack.md | 6 + package-lock.json | 13 +- plugins/webhook-egress/src/webhook.egress.ts | 32 +--- utils/response-rejection/package.json | 3 +- utils/response-rejection/src/index.ts | 157 +++++++++++++++---- 5 files changed, 148 insertions(+), 63 deletions(-) create mode 100644 .changeset/empty-crews-attack.md diff --git a/.changeset/empty-crews-attack.md b/.changeset/empty-crews-attack.md new file mode 100644 index 000000000..624a8bab8 --- /dev/null +++ b/.changeset/empty-crews-attack.md @@ -0,0 +1,6 @@ +--- +'@flatfile/util-response-rejection': patch +'@flatfile/plugin-webhook-egress': patch +--- + +Upgrade to the @flatfile/util-response-rejection plugin to support deleting successfully submitted records or adding a status column to indicate successful/rejected records. diff --git a/package-lock.json b/package-lock.json index c5ed6c4f9..39dee51f5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13215,15 +13215,15 @@ }, "plugins/yaml-schema": { "name": "@flatfile/plugin-convert-yaml-schema", - "version": "0.0.1", + "version": "0.0.2", "license": "ISC", "dependencies": { "@flatfile/api": "^1.5.33", "@flatfile/plugin-convert-json-schema": "^0.0.4", "@flatfile/plugin-json-schema": "^0.0.2", "@flatfile/plugin-space-configure": "^0.1.5", - "@flatfile/util-fetch-schema": "^0.0.1", - "@flatfile/utils-testing": "^0.0.5", + "@flatfile/util-fetch-schema": "^0.0.2", + "@flatfile/utils-testing": "^0.0.6", "@hyperjump/json-schema": "^1.6.4", "axios": "^1.5.1", "js-yaml": "^4.1.0" @@ -13308,7 +13308,7 @@ }, "utils/fetch-schema": { "name": "@flatfile/util-fetch-schema", - "version": "0.0.1", + "version": "0.0.2", "license": "ISC", "dependencies": { "@flatfile/api": "^1.5.30", @@ -13336,7 +13336,8 @@ "version": "0.1.4", "license": "ISC", "dependencies": { - "@flatfile/api": "^1.5.37" + "@flatfile/api": "^1.5.37", + "@flatfile/util-common": "^0.2.3" }, "engines": { "node": ">= 16" @@ -13344,7 +13345,7 @@ }, "utils/testing": { "name": "@flatfile/utils-testing", - "version": "0.0.5", + "version": "0.0.6", "license": "ISC", "dependencies": { "@flatfile/api": "^1.5.37", diff --git a/plugins/webhook-egress/src/webhook.egress.ts b/plugins/webhook-egress/src/webhook.egress.ts index 2dd00399f..6e992d244 100644 --- a/plugins/webhook-egress/src/webhook.egress.ts +++ b/plugins/webhook-egress/src/webhook.egress.ts @@ -2,11 +2,11 @@ import api from '@flatfile/api' import { FlatfileListener } from '@flatfile/listener' import { jobHandler } from '@flatfile/plugin-job-handler' import { logError } from '@flatfile/util-common' +import axios from 'axios' import { - ResponseRejection, + RejectionResponse, responseRejectionHandler, -} from '@flatfile/util-response-rejection' -import axios from 'axios' +} from '../../../utils/response-rejection/src' // TODO: replace with '@flatfile/util-response-rejection' export function webhookEgress(job: string, webhookUrl?: string) { return function (listener: FlatfileListener) { @@ -48,22 +48,11 @@ export function webhookEgress(job: string, webhookUrl?: string) { ) if (response.status === 200) { - const rejections: ResponseRejection = response.data.rejections + const rejections: RejectionResponse = response.data.rejections if (rejections) { - const totalRejectedRecords = await responseRejectionHandler( - rejections - ) - return { - outcome: { - next: { - type: 'id', - id: rejections.id, - label: 'See rejections', - }, - message: `${totalRejectedRecords} record(s) were rejected during data submission. Review the rejection notes, fix, then resubmit.`, - }, - } + return await responseRejectionHandler(rejections) } + return { outcome: { message: `Data was successfully submitted to the provided webhook. Go check it out at ${webhookReceiver}.`, @@ -85,13 +74,8 @@ export function webhookEgress(job: string, webhookUrl?: string) { '@flatfile/plugin-webhook-egress', JSON.stringify(error, null, 2) ) - - return { - outcome: { - message: - "This job failed probably because it couldn't find the webhook URL.", - }, - } + // Throw error to fail job + throw new Error(`Error posting data to webhook`) } }) ) diff --git a/utils/response-rejection/package.json b/utils/response-rejection/package.json index 34f202fa4..0ef558578 100644 --- a/utils/response-rejection/package.json +++ b/utils/response-rejection/package.json @@ -27,6 +27,7 @@ }, "license": "ISC", "dependencies": { - "@flatfile/api": "^1.5.37" + "@flatfile/api": "^1.5.37", + "@flatfile/util-common": "^0.2.3" } } diff --git a/utils/response-rejection/src/index.ts b/utils/response-rejection/src/index.ts index 02f5ecd57..727f9444d 100644 --- a/utils/response-rejection/src/index.ts +++ b/utils/response-rejection/src/index.ts @@ -1,63 +1,156 @@ import api, { Flatfile } from '@flatfile/api' +import { processRecords } from '@flatfile/util-common' -export interface ResponseRejection { +export interface RejectionResponse { id: string - sheets: ResponseRejectionSheet[] + message?: string + deleteSubmitted?: boolean + sheets: SheetRejections[] } -export interface ResponseRejectionSheet { - id: string - name?: string - rejectedRecords: ResponseRejectionRecord[] +export interface SheetRejections { + sheetId: string + rejectedRecords: RecordRejections[] } -export interface ResponseRejectionRecord { +export interface RecordRejections { id: string values: { field: string; message: string }[] } export async function responseRejectionHandler( - responseRejection: ResponseRejection -): Promise { + responseRejection: RejectionResponse +): Promise { let totalRejectedRecords = 0 - const results = await Promise.all( - responseRejection.sheets.map((sheet) => updateSheet(sheet)) - ) - totalRejectedRecords = results.reduce((acc, val) => acc + val, 0) + // Using a for...of loop to handle asynchronous operations + for (const sheet of responseRejection.sheets || []) { + const count = await updateSheet(sheet, responseRejection.deleteSubmitted) + totalRejectedRecords += count + } + + const message = responseRejection.message ?? getMessage(totalRejectedRecords) + const next = getNext(totalRejectedRecords) + + return { + outcome: { + heading: totalRejectedRecords > 0 ? 'Rejected Records' : 'Success!', + acknowledge: true, + ...(next && { next }), + message, + }, + } +} + +function getMessage(totalRejectedRecords) { + return totalRejectedRecords > 0 + ? `${totalRejectedRecords} record(s) were rejected during data submission. Review the rejection notes, fix, then resubmit.` + : 'Data was successfully submitted.' +} - return totalRejectedRecords +function getNext(totalRejectedRecords): Flatfile.JobOutcomeNext | undefined { + return totalRejectedRecords > 0 + ? { + type: 'url', + url: '?filter=error', + label: 'See rejections', + } + : undefined } -async function updateSheet(sheet: ResponseRejectionSheet): Promise { - if (!sheet.rejectedRecords?.length) { - return 0 +async function updateSheet( + sheetRejections: SheetRejections, + deleteSubmitted: boolean +): Promise { + if (!deleteSubmitted) { + await addSubmissionStatusField(sheetRejections.sheetId) } - const rejectedRecordsIds = sheet.rejectedRecords.map((record) => record.id) - const sheetRecords = await api.records.get(sheet.id) + await processRecords(sheetRejections.sheetId, async (records) => { + await updateRecords(sheetRejections, records, deleteSubmitted) + }) - const rejectedSheetRecords: Flatfile.Record_[] = - sheetRecords.data.records?.filter((record: Flatfile.Record_) => - rejectedRecordsIds.includes(record.id) - ) + return sheetRejections.rejectedRecords.length +} + +async function addSubmissionStatusField(sheetId: string): Promise { + try { + const { data: sheet } = await api.sheets.get(sheetId) + if ( + !sheet.config.fields.some((field) => field.key === 'submissionStatus') + ) { + await api.sheets.addField(sheet.id, { + key: 'submissionStatus', + label: 'Submission Status', + type: 'enum', + readonly: true, + config: { + allowCustom: false, + options: [ + { label: 'Rejected', value: 'rejected' }, + { label: 'Submitted', value: 'submitted' }, + ], + }, + }) + } + } catch (error) { + console.error('Error adding rejection status field:', error) + throw 'Error adding rejection status field' + } +} - for (const record of rejectedSheetRecords || []) { - const rejectedRecord: ResponseRejectionRecord = sheet.rejectedRecords.find( +async function updateRecords( + rejections: SheetRejections, + records: Flatfile.RecordsWithLinks, + deleteSubmitted: boolean +): Promise { + records.forEach((record) => { + const rejectedRecord = rejections.rejectedRecords.find( (item) => item.id === record.id ) - for (const value of rejectedRecord.values) { + + rejectedRecord?.values.forEach((value) => { if (record.values[value.field]) { record.values[value.field].messages = [ - { - type: 'error', - message: value.message, - }, + { type: 'error', message: value.message }, ] } + }) + + if (!deleteSubmitted) { + record.values['submissionStatus'].value = rejectedRecord + ? 'rejected' + : 'submitted' } + }) + + try { + await api.records.update(rejections.sheetId, records) + } catch (error) { + console.error('Error updating records:', error) + throw new Error('Error updating records') + } + + if (deleteSubmitted && records.length !== rejections.rejectedRecords.length) { + await deleteValidRecords(rejections.sheetId) } +} - await api.records.update(sheet.id, rejectedSheetRecords) - return rejectedRecordsIds.length +async function deleteValidRecords(sheetId: string): Promise { + try { + const { data: sheet } = await api.sheets.get(sheetId) + await api.jobs.create({ + type: 'workbook', + operation: 'delete-records', + trigger: 'immediate', + source: sheet.workbookId, + config: { + sheet: sheetId, + filter: 'valid', + }, + }) + } catch (error) { + console.error('Error deleting all records:', error) + throw new Error('Error deleting all records') + } }