feat: zip egress plugin

.changeset/dry-apples-teach.md

@@ -0,0 +1,5 @@
+---
+'@flatfile/plugin-zip-egress': minor
+---
+
+Introducing @flatfile/plugin-zip-egress, a plugin for egressing a Flatfile Workbook as a collection of zipped CSV files.

plugins/zip-egress/README.md

@@ -0,0 +1,22 @@
+# @flatfile/plugin-zip-egress
+
+This plugin is designed to egress a Flatfile Workbook as a collection of .csv files within a .zip archive. 
+
+## Usage
+
+Install the plugin:
+
+```bash
+npm i @flatfile/plugin-zip-egress
+```
+
+Plugin setup in `listener.ts` file:
+
+```typescript
+import type { FlatfileListener } from '@flatfile/listener'
+import { zipEgressPlugin } from '@flatfile/plugin-zip-egress'
+
+export default function (listener: FlatfileListener) {
+  listener.use(zipEgressPlugin('workbook:downloadWorkbook'))
+}
+```

plugins/zip-egress/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@flatfile/plugin-zip-egress",
+  "version": "0.0.0",
+  "description": "A plugin for egressing a Flatfile Workbook as a collection of zipped CSV files.",
+  "registryMetadata": {
+    "category": "egress"
+  },
+  "engines": {
+    "node": ">= 16"
+  },
+  "source": "src/index.ts",
+  "main": "dist/main.js",
+  "module": "dist/module.mjs",
+  "types": "dist/types.d.ts",
+  "scripts": {
+    "build": "parcel build",
+    "dev": "parcel watch",
+    "check": "tsc ./**/*.ts --noEmit --esModuleInterop",
+    "test": "jest --passWithNoTests"
+  },
+  "keywords": [],
+  "author": "Flatfile, Inc.",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/FlatFilers/flatfile-plugins.git",
+    "directory": "plugins/zip-egress"
+  },
+  "license": "ISC",
+  "dependencies": {
+    "@flatfile/api": "^1.7.4",
+    "@flatfile/listener": "^1.0.1",
+    "@flatfile/plugin-job-handler": "^0.4.1",
+    "@flatfile/util-common": "^1.0.0",
+    "adm-zip": "^0.5.12",
+    "csv-stringify": "^6.4.6"
+  },
+  "peerDependencies": {
+    "@flatfile/listener": "^1.0.1"
+  }
+}

plugins/zip-egress/src/index.ts

@@ -0,0 +1 @@
+export * from './zip.egress'

plugins/zip-egress/src/utils.ts

@@ -0,0 +1,3 @@
+export const castBoolean = (value) => {
+  return value ? 'true' : 'false'
+}

plugins/zip-egress/src/zip.egress.ts

@@ -0,0 +1,116 @@
+import type { Flatfile } from '@flatfile/api'
+import api from '@flatfile/api'
+import type { FlatfileEvent } from '@flatfile/listener'
+import { jobHandler } from '@flatfile/plugin-job-handler'
+import { processRecords } from '@flatfile/util-common'
+import AdmZip from 'adm-zip'
+import { ColumnOption, stringify } from 'csv-stringify/sync'
+import fs from 'fs'
+import path from 'path'
+import { castBoolean } from './utils'
+
+export const zipEgressPlugin = (job, opts: PluginOptions = {}) => {
+  return jobHandler(job, async (event: FlatfileEvent, tick) => {
+    const { workbookId, spaceId, environmentId } = event.context
+    const timestamp = new Date().toISOString()
+    await tick(0, 'Preparing workbook...')
+
+    try {
+      const { data: workbook } = await api.workbooks.get(workbookId)
+      const sheets = workbook.sheets.filter(
+        (sheet) => !opts.excludedSheets?.includes(sheet.config.slug)
+      )
+
+      const zip = new AdmZip()
+      let i = 0
+      for (const sheet of sheets) {
+        const { fields } = sheet.config
+        const columns: ColumnOption[] = []
+        fields.forEach((field) => {
+          if (!!field.metadata?.excludeFromExport) return
+          columns.push({ key: field.key, header: field.label })
+        })
+
+        const csvFilePath = path.join(
+          '/tmp',
+          `${sheet.config.slug}-${timestamp}.csv`
+        )
+        fs.closeSync(fs.openSync(csvFilePath, 'w'))
+        await processRecords(
+          sheet.id,
+          async (records, pageNumber) => {
+            let normalizedRecords = records.map(({ values }) => {
+              const result = fields.reduce((acc, { key }) => {
+                acc[key] = values[key] ? values[key].value : ''
+                return acc
+              }, {})
+              return result
+            })
+
+            if (pageNumber === 1 && records?.length === 0) {
+              const emptyRecord = fields.reduce(
+                (acc, { key }) => ({ ...acc, [key]: '' }),
+                {}
+              )
+              normalizedRecords = [emptyRecord]
+            }
+            const rows = stringify(normalizedRecords, {
+              header: pageNumber === 1,
+              columns: columns,
+              cast: { boolean: castBoolean },
+            })
+
+            await fs.promises.appendFile(csvFilePath, rows)
+          },
+          opts.getRecordsRequest
+        )
+
+        zip.addLocalFile(csvFilePath)
+
+        await tick(
+          10 + Math.round(((i + 1) / sheets.length) * 70),
+          `${sheet.name} Prepared`
+        )
+        await fs.promises.unlink(csvFilePath)
+        i++
+      }
+
+      await tick(81, `Uploading file...`)
+
+      const zipFilePath = path.join('/tmp', `${workbook.name}-${timestamp}.zip`)
+      zip.writeZip(zipFilePath)
+      const stream = fs.createReadStream(zipFilePath)
+
+      await api.files.upload(stream, {
+        spaceId,
+        environmentId,
+        mode: 'export',
+      })
+
+      await fs.promises.unlink(zipFilePath)
+
+      return {
+        outcome: {
+          acknowledge: true,
+          message: 'Successfully exported workbook',
+          next: {
+            type: 'id',
+            id: spaceId,
+            path: 'files',
+            query: 'mode=export',
+            label: 'See all downloads',
+          },
+        },
+      }
+    } catch (error) {
+      console.error(error)
+      throw new Error('Failed to export workbook')
+    }
+  })
+}
+
+export interface PluginOptions {
+  readonly excludedSheets?: string[]
+  readonly getRecordsRequest?: Flatfile.GetRecordsRequest
+  readonly debug?: boolean
+}