exporter: data masked sheet generator

package.json

@@ -10,7 +10,8 @@
     "flatfilers/*",
     "plugins/*",
     "support/*",
-    "utils/*"
+    "utils/*",
+    "validators/*"
   ],
   "scripts": {
     "clean": "find ./ '(' -name 'node_modules' -o -name 'dist' -o -name '.turbo' -o -name '.parcel-cache' ')' -type d -exec rm -rf {} +",

validators/DataMaskingSheetGenerator/README.MD

@@ -0,0 +1,79 @@
+# Data Masking Sheet Generator
+
+A Flatfile plugin that automatically creates a masked version of a sheet, applying customizable data masking rules to sensitive columns.
+
+## Features
+
+- Automatically creates a new sheet with masked data
+- Supports multiple masking techniques:
+  - Hashing
+  - Partial masking
+  - Tokenization
+  - PII masking
+- Customizable masking rules
+- Caches masked values for efficiency
+- Adds metadata to the masked sheet for traceability
+
+## Installation
+
+To install the Data Masking Sheet Generator plugin, use npm:
+
+```bash
+npm install @flatfile/plugin-data-masking-sheet-generator
+```
+
+## Example Usage
+
+```javascript
+import { FlatfileListener } from "@flatfile/listener";
+import DataMaskingSheetGenerator from "@flatfile/plugin-data-masking-sheet-generator";
+
+const listener = new FlatfileListener();
+
+listener.use(DataMaskingSheetGenerator);
+
+listener.configure({
+  recordHooks: {
+    "records:created": async (record, event) => {
+      // Your existing record hook logic
+      return record;
+    },
+  },
+});
+```
+
+## Configuration
+
+The plugin can be configured by passing options in the event payload:
+
+```javascript
+event.payload = {
+  columnsToMask: ["email", "phone", "ssn"],
+  maskingRules: {
+    email: { type: "hash" },
+    phone: { type: "partial", options: { showLastDigits: 4 } },
+    ssn: { type: "tokenize", options: { tokenLength: 10 } },
+  },
+};
+```
+
+### Default Masking Rules
+
+The plugin comes with default masking rules for common data types:
+
+- email: hashed
+- phone: partially masked (last 4 digits visible)
+- ssn: partially masked (last 4 digits visible)
+- creditCard: partially masked (last 4 digits visible)
+- name: tokenized (8 characters)
+- address: PII masked
+
+## Behavior
+
+1. When the 'records:created' event is triggered, the plugin creates a new sheet named "{OriginalSheetName} (Masked)".
+2. It applies the specified masking rules to the columns defined in `columnsToMask`.
+3. The plugin caches masked values to improve performance for repeated values.
+4. Masked records are inserted into the new sheet.
+5. Metadata about the masking process is added to the new sheet.
+
+Note: If an error occurs during the masking process for a specific value, it will be replaced with '[MASKING_ERROR]'.

validators/DataMaskingSheetGenerator/metadata.json

@@ -0,0 +1,77 @@
+{
+  "timestamp": "2024-09-25T06-11-27-443Z",
+  "task": "Create a Data Masking Sheet Generator Flatfile Listener plugin:\n      - Implement a custom action to create a new Sheet with masked data from an existing Sheet\n      - Allow users to select the source Sheet and specify which columns to mask\n      - Implement various masking techniques (e.g., hashing, partial masking, tokenization) for different data types\n      - Preserve the original data structure and column names in the new Sheet\n      - Provide options for configurable masking rules (e.g., show last 4 digits of credit card numbers)\n      - Implement consistent masking for repeated values within the same column\n      - Handle sensitive data types like PII (Personally Identifiable Information) with appropriate masking methods\n      - Implement error handling for unsupported data types or masking failures\n      - Add metadata to the new Sheet indicating it contains masked data\n      - Use the least amount of steps as possible",
+  "summary": "Based on the Event Topics verification, the DataMaskingSheetGenerator plugin needs to be adjusted to use valid event topics. The plugin will be modified to use the 'records:created' event topic instead of the invalid 'dataMasking' topic. The code has been optimized and finalized to meet all requirements.",
+  "steps": [
+    [
+      "Retrieve information about Flatfile Listeners and the Record Hook plugin to understand the structure and capabilities we can leverage for our data masking plugin.\n",
+      "#E1",
+      "PineconeAssistant",
+      "Provide information about Flatfile Listeners and the Record Hook plugin, including their structure and capabilities for data manipulation",
+      "Plan: Retrieve information about Flatfile Listeners and the Record Hook plugin to understand the structure and capabilities we can leverage for our data masking plugin.\n#E1 = PineconeAssistant[Provide information about Flatfile Listeners and the Record Hook plugin, including their structure and capabilities for data manipulation]"
+    ],
+    [
+      "Based on the retrieved information, create a skeleton for the Data Masking Sheet Generator plugin, including the necessary imports and the main listener function.\n",
+      "#E2",
+      "LLM",
+      "Create a skeleton for a Flatfile Listener plugin named DataMaskingSheetGenerator, using the information from #E1. Include necessary imports and a main listener function",
+      "Plan: Based on the retrieved information, create a skeleton for the Data Masking Sheet Generator plugin, including the necessary imports and the main listener function.\n#E2 = LLM[Create a skeleton for a Flatfile Listener plugin named DataMaskingSheetGenerator, using the information from #E1. Include necessary imports and a main listener function]"
+    ],
+    [
+      "Implement the custom action to create a new Sheet with masked data from an existing Sheet, allowing users to select the source Sheet and specify columns to mask.\n",
+      "#E3",
+      "LLM",
+      "Extend the DataMaskingSheetGenerator plugin from #E2 to include a custom action for creating a new Sheet with masked data. Implement logic for selecting the source Sheet and specifying columns to mask",
+      "Plan: Implement the custom action to create a new Sheet with masked data from an existing Sheet, allowing users to select the source Sheet and specify columns to mask.\n#E3 = LLM[Extend the DataMaskingSheetGenerator plugin from #E2 to include a custom action for creating a new Sheet with masked data. Implement logic for selecting the source Sheet and specifying columns to mask]"
+    ],
+    [
+      "Implement various masking techniques (hashing, partial masking, tokenization) for different data types, and add configurable masking rules.\n",
+      "#E4",
+      "LLM",
+      "Add masking functions to the DataMaskingSheetGenerator plugin from #E3, including hashing, partial masking, and tokenization. Implement configurable masking rules, such as showing the last 4 digits of credit card numbers",
+      "Plan: Implement various masking techniques (hashing, partial masking, tokenization) for different data types, and add configurable masking rules.\n#E4 = LLM[Add masking functions to the DataMaskingSheetGenerator plugin from #E3, including hashing, partial masking, and tokenization. Implement configurable masking rules, such as showing the last 4 digits of credit card numbers]"
+    ],
+    [
+      "Implement consistent masking for repeated values within the same column and handle sensitive data types like PII with appropriate masking methods.\n",
+      "#E5",
+      "LLM",
+      "Extend the DataMaskingSheetGenerator plugin from #E4 to include consistent masking for repeated values and special handling for PII data",
+      "Plan: Implement consistent masking for repeated values within the same column and handle sensitive data types like PII with appropriate masking methods.\n#E5 = LLM[Extend the DataMaskingSheetGenerator plugin from #E4 to include consistent masking for repeated values and special handling for PII data]"
+    ],
+    [
+      "Implement error handling for unsupported data types or masking failures, and add metadata to the new Sheet indicating it contains masked data.\n",
+      "#E6",
+      "LLM",
+      "Add error handling and metadata functionality to the DataMaskingSheetGenerator plugin from #E5",
+      "Plan: Implement error handling for unsupported data types or masking failures, and add metadata to the new Sheet indicating it contains masked data.\n#E6 = LLM[Add error handling and metadata functionality to the DataMaskingSheetGenerator plugin from #E5]"
+    ],
+    [
+      "Review and optimize the complete DataMaskingSheetGenerator plugin, ensuring all requirements are met and the code is efficient.\n",
+      "#E7",
+      "LLM",
+      "Review and optimize the complete DataMaskingSheetGenerator plugin from #E6, ensuring all requirements are met and the code is efficient. Remove any unused imports and validate that all params are correct for the plugins used",
+      "Plan: Review and optimize the complete DataMaskingSheetGenerator plugin, ensuring all requirements are met and the code is efficient.\n#E7 = LLM[Review and optimize the complete DataMaskingSheetGenerator plugin from #E6, ensuring all requirements are met and the code is efficient. Remove any unused imports and validate that all params are correct for the plugins used]"
+    ],
+    [
+      "Verify that the listener subscribes to valid Event Topics using the event.topics.fact file.\n",
+      "#E8",
+      "PineconeAssistant",
+      "Verify that the Event Topics used in the DataMaskingSheetGenerator plugin from #E7 are valid according to the event.topics.fact file",
+      "Plan: Verify that the listener subscribes to valid Event Topics using the event.topics.fact file.\n#E8 = PineconeAssistant[Verify that the Event Topics used in the DataMaskingSheetGenerator plugin from #E7 are valid according to the event.topics.fact file]"
+    ],
+    [
+      "Make any necessary adjustments based on the Event Topics verification and finalize the DataMaskingSheetGenerator plugin.\n",
+      "#E9",
+      "LLM",
+      "Adjust the DataMaskingSheetGenerator plugin from #E7 based on the Event Topics verification in #E8, and finalize the code",
+      "Plan: Make any necessary adjustments based on the Event Topics verification and finalize the DataMaskingSheetGenerator plugin.\n#E9 = LLM[Adjust the DataMaskingSheetGenerator plugin from #E7 based on the Event Topics verification in #E8, and finalize the code]"
+    ]
+  ],
+  "metrics": {
+    "tokens": {
+      "plan": 4740,
+      "state": 5712,
+      "total": 10452
+    }
+  }
+}

validators/DataMaskingSheetGenerator/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "@flatfile/plugin-data-masking",
+  "version": "1.0.0",
+  "description": "A Flatfile plugin for data masking and PII protection",
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "browser": {
+    "./dist/index.js": "./dist/index.browser.js",
+    "./dist/index.mjs": "./dist/index.browser.mjs"
+  },
+  "exports": {
+    "types": "./dist/index.d.ts",
+    "node": {
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    },
+    "browser": {
+      "require": "./dist/index.browser.js",
+      "import": "./dist/index.browser.mjs"
+    },
+    "default": "./dist/index.mjs"
+  },
+  "source": "./src/index.ts",
+  "files": [
+    "dist/**"
+  ],
+  "scripts": {
+    "build": "rollup -c",
+    "build:watch": "rollup -c --watch",
+    "build:prod": "NODE_ENV=production rollup -c",
+    "check": "tsc ./**/*.ts --noEmit --esModuleInterop",
+    "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand"
+  },
+  "keywords": [
+    "flatfile",
+    "plugin",
+    "data-masking",
+    "pii-protection",
+    "flatfile-plugins",
+    "category-transform"
+  ],
+  "author": "Your Name",
+  "license": "MIT",
+  "dependencies": {
+    "@flatfile/plugin-record-hook": "^1.7.0",
+    "@flatfile/api": "^1.9.15"
+  },
+  "peerDependencies": {
+    "@flatfile/listener": "^1.0.5"
+  },
+  "devDependencies": {
+    "@flatfile/hooks": "^1.5.0",
+    "@flatfile/rollup-config": "^0.1.1",
+    "@types/node": "^22.7.0",
+    "typescript": "^5.6.2"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/FlatFilers/flatfile-plugins.git",
+    "directory": "plugins/data-masking"
+  },
+  "browserslist": [
+    "> 0.5%",
+    "last 2 versions",
+    "not dead"
+  ]
+}

validators/DataMaskingSheetGenerator/rollup.config.mjs

@@ -0,0 +1,26 @@
+import { buildConfig } from '@flatfile/rollup-config';
+
+const umdExternals = [
+  '@flatfile/api',
+  '@flatfile/hooks',
+  '@flatfile/listener',
+  '@flatfile/util-common',
+  '@flatfile/plugin-record-hook',
+  'crypto'
+];
+
+const config = buildConfig({
+  input: 'src/index.ts', // Assuming your main file is src/index.ts
+  includeUmd: true,
+  umdConfig: { 
+    name: 'DataMaskingSheetGenerator', 
+    external: umdExternals 
+  },
+  external: [
+    ...umdExternals,
+    'crypto'
+  ],
+  includeBrowser: true, // Include browser build
+});
+
+export default config;