Configure Space with YAML Plugin

.changeset/yellow-rats-enjoy.md

@@ -0,0 +1,8 @@
+---
+'@flatfile/plugin-convert-json-schema': patch
+'@flatfile/plugin-convert-yaml-schema': patch
+'@flatfile/util-fetch-schema': patch
+'@flatfile/utils-testing': patch
+---
+
+DRY up YAML plugin, remove accidental edits to JSON plugin

.changeset/young-bottles-thank.md

@@ -0,0 +1,5 @@
+---
+'@flatfile/plugin-convert-yaml-schema': patch
+---
+
+Introducing the @flatfile/plugin-convert-yaml-schema plugin to configure Flatfile Spaces based on a provided YAML Schema.

.prettierrc

@@ -3,5 +3,13 @@
   "singleQuote": true,
   "jsxSingleQuote": true,
   "trailingComma": "es5",
-  "semi": false
-}
+  "semi": false,
+  "overrides": [
+    {
+      "files": "*.yml",
+      "options": {
+        "parser": "yaml"
+      }
+    }
+  ]
+}

plugins/json-schema/package.json

@@ -29,10 +29,11 @@
   "dependencies": {
     "@flatfile/api": "^1.5.33",
     "@flatfile/plugin-space-configure": "^0.1.5",
+    "@flatfile/util-fetch-schema": "^0.0.1",
     "@hyperjump/json-schema": "^1.6.4",
     "axios": "^1.5.1"
   },
   "devDependencies": {
     "express": "^4.18.2"
   }
-}
+}

plugins/yaml-schema/CHANGELOG.md

@@ -0,0 +1,5 @@
+# @flatfile/plugin-yaml-schema
+
+## 0.0.1
+
+### Patch Changes

plugins/yaml-schema/README.md

@@ -0,0 +1,7 @@
+# @flatfile/plugin-convert-yaml-schema
+
+This package automatically converts YAML Schema to the Flatfile Blueprint, a powerful DDL (Data Definition Language) created by Flatfile with a focus on validation and data preparation.
+
+## Get Started
+
+Follow [this guide](https://flatfile.com/docs/plugins/schemas/convert-yaml-schema) to learn how to use the plugin.

plugins/yaml-schema/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@flatfile/plugin-convert-yaml-schema",
+  "version": "0.0.1",
+  "description": "A plugin for converting YAML Schema definitions to Flatfile Blueprint.",
+  "registryMetadata": {
+    "category": "schema-converters"
+  },
+  "engines": {
+    "node": ">= 16"
+  },
+  "source": "src/index.ts",
+  "main": "dist/main.js",
+  "module": "dist/module.mjs",
+  "types": "dist/types.d.ts",
+  "scripts": {
+    "build": "parcel build",
+    "dev": "parcel watch",
+    "check": "tsc ./**/*.ts --noEmit --esModuleInterop",
+    "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand"
+  },
+  "keywords": [],
+  "author": "Flatfile, Inc.",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/FlatFilers/flatfile-plugins.git",
+    "directory": "plugins/yaml-schema"
+  },
+  "license": "ISC",
+  "dependencies": {
+    "@flatfile/api": "^1.5.33",
+    "@flatfile/plugin-json-schema": "^0.0.2",
+    "@flatfile/plugin-space-configure": "^0.1.5",
+    "@hyperjump/json-schema": "^1.6.4",
+    "axios": "^1.5.1",
+    "js-yaml": "^4.1.0"
+  },
+  "devDependencies": {
+    "express": "^4.18.2"
+  }
+}

plugins/yaml-schema/src/index.ts

@@ -0,0 +1,37 @@
+import { Flatfile } from '@flatfile/api'
+import { FlatfileEvent, FlatfileListener } from '@flatfile/listener'
+import { generateSetup } from '@flatfile/plugin-convert-json-schema'
+import { configureSpace } from '@flatfile/plugin-space-configure'
+import type {
+  ModelToSheetConfig,
+  PartialWorkbookConfig,
+} from '@flatfile/util-fetch-schema'
+import { getSchemas } from '@flatfile/util-fetch-schema/src'
+import jsYaml from 'js-yaml'
+
+export function configureSpaceWithYamlSchema(
+  models?: ModelToSheetConfig[],
+  options?: {
+    workbookConfig?: PartialWorkbookConfig
+    debug?: boolean
+  },
+  callback?: (
+    event: FlatfileEvent,
+    workbookIds: string[],
+    tick: (progress?: number, message?: string) => Promise<Flatfile.JobResponse>
+  ) => any | Promise<any>
+) {
+  return async function (listener: FlatfileListener) {
+    const schemas = await getSchemas(models)
+    listener.use(
+      configureSpace(
+        await generateSetup(
+          models,
+          schemas.map((schema) => jsYaml.load(schema)),
+          options
+        ),
+        callback
+      )
+    )
+  }
+}

plugins/yaml-schema/tests/exampleData.yml

@@ -0,0 +1,29 @@
+$id: 'https://localhost:3000/json'
+description: 'A basic set of JSON Schema to test data type conversions simply'
+type: 'object'
+title: 'ExampleData'
+properties:
+  stringColumn:
+    description: 'A column for strings!'
+    type: 'string'
+
+  integerColumn:
+    description: 'A column for integers!'
+    type: 'integer'
+
+  arrayColumn:
+    description: 'A column for string arrays!'
+    type: 'array'
+    items:
+      type: 'string'
+
+  objectColumn:
+    description: 'A column for nested columns containing numbers and strings!'
+    type: 'object'
+    properties:
+      nestedUniqueNumberColumn:
+        description: 'A column for unique numbers!'
+        type: 'number'
+        uniqueItems: true
+      nestedStringColumn:
+        type: 'string'

plugins/yaml-schema/tests/yaml-schema.e2e.spec.ts

@@ -0,0 +1,99 @@
+import api from '@flatfile/api'
+import { deleteSpace, setupListener, setupSpace, startServer, stopServer } from '@flatfile/utils-testing'
+import express from 'express'
+import fs from 'fs'
+import path from 'path'
+import { configureSpaceWithYamlSchema } from '../src'
+
+const app = express()
+const port = 8080
+const url = `http://localhost:${port}/`
+
+let server
+
+describe('configureSpaceWithJsonSchema() e2e', () => {
+  const pureDataSchema = fs.readFileSync(
+    path.resolve(__dirname, './exampleData.yml'),
+    'utf-8'
+  )
+
+  const expectedWorkbookData = {
+    name: 'JSON Schema Workbook',
+    labels: [],
+    sheets: [
+      {
+        name: 'ExampleData',
+        config: {
+          name: 'ExampleData',
+          description:
+            'A basic set of JSON Schema to test data type conversions simply',
+          fields: [
+            {
+              type: 'string',
+              key: 'stringColumn',
+              label: 'stringColumn',
+              description: 'A column for strings!',
+            },
+            {
+              type: 'number',
+              key: 'integerColumn',
+              label: 'integerColumn',
+              description: 'A column for integers!',
+            },
+            {
+              type: 'enum',
+              config: { options: [] },
+              key: 'arrayColumn',
+              label: 'arrayColumn',
+              description: 'An enum of Selected Values',
+            },
+            {
+              type: 'number',
+              key: 'objectColumn_nestedUniqueNumberColumn',
+              label: 'objectColumn_nestedUniqueNumberColumn',
+              description: 'A column for unique numbers!',
+            },
+            {
+              type: 'string',
+              key: 'objectColumn_nestedStringColumn',
+              label: 'objectColumn_nestedStringColumn',
+            },
+          ],
+        },
+      },
+    ],
+  }
+
+  let spaceId: string
+  const listener = setupListener()
+
+  beforeAll(async () => {
+    console.log(`Starting temporary server on port ${port}`)
+    server = startServer(app, port, pureDataSchema)
+    console.log('Setting up Space and Retrieving spaceId')
+
+    await listener.use(configureSpaceWithYamlSchema([{ sourceUrl: url }]))
+
+    const space = await setupSpace()
+    spaceId = space.id
+  })
+
+  afterAll(async () => {
+    stopServer(server)
+    await deleteSpace(spaceId)
+  })
+
+  it('should configure a space and correctly format and flatten the JSON Schema', async () => {
+    console.log('starting configuration')
+    await listener.waitFor('job:ready', 1, 'space:configure')
+
+    const space = await api.spaces.get(spaceId)
+    const workspace = await api.workbooks.get(space.data.primaryWorkbookId)
+    const workspaceData = workspace.data
+
+    expect(workspaceData.name).toBe(expectedWorkbookData.name)
+    expect(workspaceData.sheets[0].config).toMatchObject(
+      expectedWorkbookData.sheets[0].config
+    )
+  })
+})

utils/fetch-schema/CHANGELOG.md

@@ -0,0 +1 @@
+# @flatfile/util-fetch-schema

utils/fetch-schema/README.md

@@ -0,0 +1,8 @@
+# @flatfile/util-fetch-schema
+
+This utility will provide a utility for fetching schemas from remote storage locations.
+
+## Get Started
+
+Follow [this guide](https://flatfile.com/docs/plugins/utilities/fetch-schema) to learn how to use the utility.
+