From fe84fe15f88cd64c0fdc1e90a85238db45c0dc74 Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Fri, 3 Nov 2023 09:26:13 -0500 Subject: [PATCH] feat: convert sql ddl -> blueprint --- package-lock.json | 104 +++++++++++++++++- .../CHANGELOG.md | 0 .../{sql-ddl => sql-ddl-converter}/README.md | 0 plugins/sql-ddl-converter/package.json | 36 ++++++ plugins/sql-ddl-converter/src/index.ts | 31 ++++++ .../sql-ddl-converter/src/setup.factory.ts | 78 +++++++++++++ plugins/sql-ddl/package.json | 17 --- plugins/sql-ddl/src/index.ts | 31 ++++++ plugins/sql-ddl/src/setup.factory.ts | 78 +++++++++++++ plugins/xlsx-extractor/src/parser.ts | 21 +++- 10 files changed, 368 insertions(+), 28 deletions(-) rename plugins/{sql-ddl => sql-ddl-converter}/CHANGELOG.md (100%) rename plugins/{sql-ddl => sql-ddl-converter}/README.md (100%) create mode 100644 plugins/sql-ddl-converter/package.json create mode 100644 plugins/sql-ddl-converter/src/index.ts create mode 100644 plugins/sql-ddl-converter/src/setup.factory.ts delete mode 100644 plugins/sql-ddl/package.json create mode 100644 plugins/sql-ddl/src/index.ts create mode 100644 plugins/sql-ddl/src/setup.factory.ts diff --git a/package-lock.json b/package-lock.json index 80523b7cf..75576e84c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -900,6 +900,10 @@ "resolved": "plugins/json-schema", "link": true }, + "node_modules/@flatfile/plugin-convert-sql-ddl": { + "resolved": "plugins/sql-ddl-converter", + "link": true + }, "node_modules/@flatfile/plugin-dedupe": { "resolved": "plugins/dedupe", "link": true @@ -948,10 +952,6 @@ "resolved": "plugins/space-configure", "link": true }, - "node_modules/@flatfile/plugin-sql-ddl": { - "resolved": "plugins/sql-ddl", - "link": true - }, "node_modules/@flatfile/plugin-tsv-extractor": { "resolved": "plugins/tsv-extractor", "link": true @@ -4979,6 +4979,11 @@ "node": ">=8" } }, + "node_modules/discontinuous-range": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/discontinuous-range/-/discontinuous-range-1.0.0.tgz", + "integrity": "sha512-c68LpLbO+7kP/b1Hr1qs8/BJ09F5khZGTxqxZuhzxpmwJKOgRFHJWIb9/KmqnqHhLdO55aOxFH/EGBvUQbL/RQ==" + }, "node_modules/dom-serializer": { "version": "1.4.1", "dev": true, @@ -8086,6 +8091,11 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==" + }, "node_modules/json-schema-ref-parser": { "version": "9.0.9", "dev": true, @@ -8489,6 +8499,11 @@ "node": ">= 8.0.0" } }, + "node_modules/moo": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/moo/-/moo-0.5.1.tgz", + "integrity": "sha512-I1mnb5xn4fO80BH9BLcF0yLypy2UKl+Cb01Fu0hJRkJjlCRtxZMWkTdAtDd5ZqCOxtCkhmRwyI57vWT+1iZ67w==" + }, "node_modules/ms": { "version": "2.1.2", "license": "MIT" @@ -8537,6 +8552,32 @@ "version": "1.4.0", "license": "MIT" }, + "node_modules/nearley": { + "version": "2.20.1", + "resolved": "https://registry.npmjs.org/nearley/-/nearley-2.20.1.tgz", + "integrity": "sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ==", + "dependencies": { + "commander": "^2.19.0", + "moo": "^0.5.0", + "railroad-diagrams": "^1.0.0", + "randexp": "0.4.6" + }, + "bin": { + "nearley-railroad": "bin/nearley-railroad.js", + "nearley-test": "bin/nearley-test.js", + "nearley-unparse": "bin/nearley-unparse.js", + "nearleyc": "bin/nearleyc.js" + }, + "funding": { + "type": "individual", + "url": "https://nearley.js.org/#give-to-nearley" + } + }, + "node_modules/nearley/node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -9372,6 +9413,23 @@ "node": ">=8" } }, + "node_modules/railroad-diagrams": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/railroad-diagrams/-/railroad-diagrams-1.0.0.tgz", + "integrity": "sha512-cz93DjNeLY0idrCNOH6PviZGRN9GJhsdm9hpn1YCS879fj4W+x5IFJhhkRZcwVgMmFF7R82UA/7Oh+R8lLZg6A==" + }, + "node_modules/randexp": { + "version": "0.4.6", + "resolved": "https://registry.npmjs.org/randexp/-/randexp-0.4.6.tgz", + "integrity": "sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ==", + "dependencies": { + "discontinuous-range": "1.0.0", + "ret": "~0.1.10" + }, + "engines": { + "node": ">=0.12" + } + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -9573,6 +9631,14 @@ "node": ">=10" } }, + "node_modules/ret": { + "version": "0.1.15", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "engines": { + "node": ">=0.12" + } + }, "node_modules/reusify": { "version": "1.0.4", "dev": true, @@ -10075,6 +10141,19 @@ "version": "1.0.3", "license": "BSD-3-Clause" }, + "node_modules/sql-ddl-to-json-schema": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/sql-ddl-to-json-schema/-/sql-ddl-to-json-schema-4.1.0.tgz", + "integrity": "sha512-Vt6j9RvkPhaTVB6fHBFaLFqTEsndABY4EznOe+iTh1pp0y5EV3m/IijqQ0N3Q4khfgGpKeJSzw9VI4CRbSwHKQ==", + "dependencies": { + "json-schema": "^0.4.0", + "moo": "0.5.1", + "nearley": "2.20.1" + }, + "engines": { + "node": ">=8.6.0 <=20" + } + }, "node_modules/srcset": { "version": "4.0.0", "dev": true, @@ -11325,7 +11404,7 @@ }, "plugins/record-hook": { "name": "@flatfile/plugin-record-hook", - "version": "1.1.9", + "version": "1.1.10", "license": "ISC", "dependencies": { "@flatfile/api": "^1.5.34", @@ -11357,8 +11436,23 @@ "plugins/sql-ddl": { "name": "@flatfile/plugin-sql-ddl", "version": "0.0.2", + "extraneous": true, "license": "ISC" }, + "plugins/sql-ddl-converter": { + "version": "0.0.2", + "license": "ISC", + "dependencies": { + "@flatfile/api": "^1.5.30", + "@flatfile/plugin-convert-json-schema": "^0.0.2", + "@flatfile/plugin-space-configure": "^0.1.5", + "axios": "^1.5.1", + "sql-ddl-to-json-schema": "^4.1.0" + }, + "engines": { + "node": ">= 16" + } + }, "plugins/tsv-extractor": { "name": "@flatfile/plugin-tsv-extractor", "version": "1.5.2", diff --git a/plugins/sql-ddl/CHANGELOG.md b/plugins/sql-ddl-converter/CHANGELOG.md similarity index 100% rename from plugins/sql-ddl/CHANGELOG.md rename to plugins/sql-ddl-converter/CHANGELOG.md diff --git a/plugins/sql-ddl/README.md b/plugins/sql-ddl-converter/README.md similarity index 100% rename from plugins/sql-ddl/README.md rename to plugins/sql-ddl-converter/README.md diff --git a/plugins/sql-ddl-converter/package.json b/plugins/sql-ddl-converter/package.json new file mode 100644 index 000000000..21efc2c24 --- /dev/null +++ b/plugins/sql-ddl-converter/package.json @@ -0,0 +1,36 @@ +{ + "name": "@flatfile/plugin-convert-sql-ddl", + "version": "0.0.2", + "description": "A plugin for converting SQL DDL into Flatfile Blueprint.", + "registryMetadata": { + "category": "schema-converters" + }, + "engines": { + "node": ">= 16" + }, + "source": "src/index.ts", + "main": "dist/main.js", + "module": "dist/module.mjs", + "types": "dist/types.d.ts", + "scripts": { + "build": "parcel build", + "dev": "parcel watch", + "check": "tsc ./**/*.ts --noEmit --esModuleInterop", + "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand" + }, + "keywords": [], + "author": "Flatfile, Inc.", + "repository": { + "type": "git", + "url": "https://github.com/FlatFilers/flatfile-plugins.git", + "directory": "plugins/sql-ddl-converter" + }, + "license": "ISC", + "dependencies": { + "@flatfile/api": "^1.5.30", + "@flatfile/plugin-convert-json-schema": "^0.0.2", + "@flatfile/plugin-space-configure": "^0.1.5", + "axios": "^1.5.1", + "sql-ddl-to-json-schema": "^4.1.0" + } +} diff --git a/plugins/sql-ddl-converter/src/index.ts b/plugins/sql-ddl-converter/src/index.ts new file mode 100644 index 000000000..f49dcf03e --- /dev/null +++ b/plugins/sql-ddl-converter/src/index.ts @@ -0,0 +1,31 @@ +import { Flatfile } from '@flatfile/api' +import { FlatfileEvent, FlatfileListener } from '@flatfile/listener' +import { configureSpace } from '@flatfile/plugin-space-configure' +import { + ModelsToSheetConfig, + PartialWorkbookConfig, + generateSetup, +} from './setup.factory' + +export function configureSpaceWithSqlDDL( + sqlDdlPath: string, + options?: { + models?: ModelsToSheetConfig + workbookConfig?: PartialWorkbookConfig + debug?: boolean + }, + callback?: ( + event: FlatfileEvent, + workbookIds: string[], + tick: (progress?: number, message?: string) => Promise + ) => any | Promise +) { + return async function (listener: FlatfileListener) { + listener.use( + configureSpace(await generateSetup(sqlDdlPath, options), callback) + ) + } +} + +export type { SetupFactory } from '@flatfile/plugin-space-configure' +export * from './setup.factory' diff --git a/plugins/sql-ddl-converter/src/setup.factory.ts b/plugins/sql-ddl-converter/src/setup.factory.ts new file mode 100644 index 000000000..8221da6f2 --- /dev/null +++ b/plugins/sql-ddl-converter/src/setup.factory.ts @@ -0,0 +1,78 @@ +import { Flatfile } from '@flatfile/api' +import { generateFields } from '@flatfile/plugin-convert-json-schema' +import { SetupFactory } from '@flatfile/plugin-space-configure' +import { Parser } from 'sql-ddl-to-json-schema' + +import { FlatfileEvent } from '@flatfile/listener' +import * as fs from 'fs' +import * as path from 'path' + +export type PartialWorkbookConfig = Omit< + Flatfile.CreateWorkbookConfig, + 'sheets' | 'name' +> & { + name?: string +} +export type PartialSheetConfig = Omit< + Flatfile.SheetConfig, + 'fields' | 'name' +> & { + name?: string +} + +export type ModelsToSheetConfig = { [key: string]: PartialSheetConfig } + +export async function generateSetup( + sqlDdlPath: string, + options?: { + models?: ModelsToSheetConfig + workbookConfig?: PartialWorkbookConfig + debug?: boolean + } +): Promise { + const sql: string = fs + .readFileSync(path.join(__dirname, '../', sqlDdlPath)) + .toString() + + const parser = new Parser('mysql') + const compactJsonTablesArray = parser.feed(sql).toCompactJson(parser.results) + // console.dir(compactJsonTablesArray, { depth: null }) + const schemas = parser + .feed(sql) + .toJsonSchemaArray({ useRef: true }, compactJsonTablesArray) + // console.dir(schemas, { depth: null }) + const sheetConfigs: Flatfile.SheetConfig[] = await Promise.all( + Object.entries(schemas) + .filter( + ([key]) => !options?.models || options?.models.hasOwnProperty(key) + ) + .map(async ([key, schema]) => { + const fields: Flatfile.Property[] = await generateFields(schema) + + const requiredFields = new Set(schema.required || []) + fields.forEach((field) => { + if (requiredFields.has(field.key)) { + field.constraints?.push({ type: 'required' }) + } + }) + + const modelDetails = options?.models?.[schema.title] + return { + name: modelDetails?.name || schema.title, + slug: modelDetails?.slug || schema.title, + fields, + ...modelDetails, + } + }) + ) + // console.dir(sheetConfigs, { depth: null }) + return { + workbooks: [ + { + name: 'Hello world', + sheets: sheetConfigs, + ...options?.workbookConfig, + }, + ], + } +} diff --git a/plugins/sql-ddl/package.json b/plugins/sql-ddl/package.json deleted file mode 100644 index a7f718c2a..000000000 --- a/plugins/sql-ddl/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@flatfile/plugin-sql-ddl", - "version": "0.0.2", - "description": "A plugin for converting SQL DDL into Flatfile blueprint.", - "registryMetadata": { - "status": "Coming soon", - "category": "schema" - }, - "keywords": [], - "author": "Alex Hollenbeck", - "repository": { - "type": "git", - "url": "https://github.com/FlatFilers/flatfile-plugins.git", - "directory": "plugins/sql-ddl" - }, - "license": "ISC" -} diff --git a/plugins/sql-ddl/src/index.ts b/plugins/sql-ddl/src/index.ts new file mode 100644 index 000000000..f49dcf03e --- /dev/null +++ b/plugins/sql-ddl/src/index.ts @@ -0,0 +1,31 @@ +import { Flatfile } from '@flatfile/api' +import { FlatfileEvent, FlatfileListener } from '@flatfile/listener' +import { configureSpace } from '@flatfile/plugin-space-configure' +import { + ModelsToSheetConfig, + PartialWorkbookConfig, + generateSetup, +} from './setup.factory' + +export function configureSpaceWithSqlDDL( + sqlDdlPath: string, + options?: { + models?: ModelsToSheetConfig + workbookConfig?: PartialWorkbookConfig + debug?: boolean + }, + callback?: ( + event: FlatfileEvent, + workbookIds: string[], + tick: (progress?: number, message?: string) => Promise + ) => any | Promise +) { + return async function (listener: FlatfileListener) { + listener.use( + configureSpace(await generateSetup(sqlDdlPath, options), callback) + ) + } +} + +export type { SetupFactory } from '@flatfile/plugin-space-configure' +export * from './setup.factory' diff --git a/plugins/sql-ddl/src/setup.factory.ts b/plugins/sql-ddl/src/setup.factory.ts new file mode 100644 index 000000000..9e375ea9a --- /dev/null +++ b/plugins/sql-ddl/src/setup.factory.ts @@ -0,0 +1,78 @@ +import { Flatfile } from '@flatfile/api' +import { SetupFactory } from '@flatfile/plugin-space-configure' +import { Parser } from 'sql-ddl-to-json-schema' +import { generateFields } from '../../json-schema/src' + +import { FlatfileEvent } from '@flatfile/listener' +import * as fs from 'fs' +import * as path from 'path' + +export type PartialWorkbookConfig = Omit< + Flatfile.CreateWorkbookConfig, + 'sheets' | 'name' +> & { + name?: string +} +export type PartialSheetConfig = Omit< + Flatfile.SheetConfig, + 'fields' | 'name' +> & { + name?: string +} + +export type ModelsToSheetConfig = { [key: string]: PartialSheetConfig } + +export async function generateSetup( + sqlDdlPath: string, + options?: { + models?: ModelsToSheetConfig + workbookConfig?: PartialWorkbookConfig + debug?: boolean + } +): Promise { + const sql: string = fs + .readFileSync(path.join(__dirname, '../', sqlDdlPath)) + .toString() + + const parser = new Parser('mysql') + const compactJsonTablesArray = parser.feed(sql).toCompactJson(parser.results) + // console.dir(compactJsonTablesArray, { depth: null }) + const schemas = parser + .feed(sql) + .toJsonSchemaArray({ useRef: true }, compactJsonTablesArray) + // console.dir(schemas, { depth: null }) + const sheetConfigs: Flatfile.SheetConfig[] = await Promise.all( + Object.entries(schemas) + .filter( + ([key]) => !options?.models || options?.models.hasOwnProperty(key) + ) + .map(async ([key, schema]) => { + const fields: Flatfile.Property[] = await generateFields(schema) + + const requiredFields = new Set(schema.required || []) + fields.forEach((field) => { + if (requiredFields.has(field.key)) { + field.constraints?.push({ type: 'required' }) + } + }) + + const modelDetails = options?.models?.[schema.title] + return { + name: modelDetails?.name || schema.title, + slug: modelDetails?.slug || schema.title, + fields, + ...modelDetails, + } + }) + ) + // console.dir(sheetConfigs, { depth: null }) + return { + workbooks: [ + { + name: 'Hello world', + sheets: sheetConfigs, + ...options?.workbookConfig, + }, + ], + } +} diff --git a/plugins/xlsx-extractor/src/parser.ts b/plugins/xlsx-extractor/src/parser.ts index 4b2efd3a8..7ccb7f93e 100644 --- a/plugins/xlsx-extractor/src/parser.ts +++ b/plugins/xlsx-extractor/src/parser.ts @@ -1,13 +1,17 @@ -import * as XLSX from 'xlsx' -import { mapKeys, mapValues } from 'remeda' -import { SheetCapture, WorkbookCapture } from '@flatfile/util-extractor' import { Flatfile } from '@flatfile/api' +import { SheetCapture, WorkbookCapture } from '@flatfile/util-extractor' +import { mapKeys, mapValues } from 'remeda' +import * as XLSX from 'xlsx' export function parseBuffer( buffer: Buffer, options?: { raw?: boolean rawNumbers?: boolean + detectHeader?: (rows: Record[]) => { + headerRow: Record + skip: number + } } ): WorkbookCapture { const workbook = XLSX.read(buffer, { @@ -19,7 +23,8 @@ export function parseBuffer( return convertSheet( value, options?.rawNumbers || false, - options?.raw || false + options?.raw || false, + options?.detectHeader || defaultDetectHeader ) }) } @@ -32,7 +37,11 @@ export function parseBuffer( function convertSheet( sheet: XLSX.WorkSheet, rawNumbers: boolean, - raw: boolean + raw: boolean, + detectHeader: (rows: Record[]) => { + headerRow: Record + skip: number + } ): SheetCapture { let rows = XLSX.utils.sheet_to_json>(sheet, { header: 'A', @@ -91,7 +100,7 @@ function prependNonUniqueHeaderColumns( const isNullOrWhitespace = (value: any) => value === null || (typeof value === 'string' && value.trim() === '') -const detectHeader = ( +const defaultDetectHeader = ( rows: Record[] ): { headerRow: Record; skip: number } => { const ROWS_TO_CHECK = 10