feat(cb2-13861): added script for generating batch plate data (#156)

package.json

@@ -45,9 +45,9 @@
     "polly-js": "^1.8.3"
   },
   "devDependencies": {
-    "@aws-sdk/client-s3": "^3.550.0",
     "@aws-sdk/client-dynamodb": "^3.359.0",
     "@aws-sdk/client-lambda": "^3.362.0",
+    "@aws-sdk/client-s3": "^3.550.0",
     "@aws-sdk/client-sns": "^3.485.0",
     "@aws-sdk/client-sqs": "^3.382.0",
     "@aws-sdk/lib-dynamodb": "^3.341.0",
@@ -78,7 +78,7 @@
     "source-map-support": "^0.5.21",
     "ts-jest": "^29.1.2",
     "ts-loader": "^9.4.2",
-    "ts-node": "^10.9.1",
+    "ts-node": "^10.9.2",
     "typescript": "^4.9.5",
     "uuid": "^8.3.2",
     "webpack": "^5.78.0",

src/handler/uploadPlateSeedData.ts

@@ -0,0 +1,44 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+/* eslint-disable @typescript-eslint/no-unsafe-call */
+/* eslint-disable @typescript-eslint/no-unsafe-assignment */
+import { marshall } from '@aws-sdk/util-dynamodb';
+import { APIGatewayProxyEvent, APIGatewayProxyResult } from 'aws-lambda';
+import 'dotenv/config';
+import * as fs from 'fs';
+import { insertBatchPlateSeedData } from '../services/database';
+import { generateBatchPlateData } from '../util/generateBatchPlateData';
+import { addHttpHeaders } from '../util/httpHeaders';
+import logger from '../util/logger';
+
+export const handler = async (event: APIGatewayProxyEvent): Promise<APIGatewayProxyResult> => {
+  logger.info('Upload plate seed end point called');
+
+  const fileNames = generateBatchPlateData();
+
+  // eslint-disable-next-line no-restricted-syntax
+  for (const fileName of fileNames) {
+    console.log(fileName);
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+    const seedData = JSON.parse(fs.readFileSync(fileName, 'utf8'));
+
+    const formattedSeedData = seedData.map((techRec: unknown) => ({
+      PutRequest: {
+        Item: marshall(techRec, { removeUndefinedValues: true }),
+      },
+    }));
+
+    const chunkSize = 25;
+
+    for (let i = 0; i < formattedSeedData.length; i += chunkSize) {
+      const chunk = formattedSeedData.slice(i, i + chunkSize);
+      // eslint-disable-next-line no-await-in-loop
+      await insertBatchPlateSeedData(chunk);
+    }
+  }
+
+  return addHttpHeaders({
+    statusCode: 200,
+    body: 'Uploaded all plate seed data',
+  });
+};

src/services/database.ts

@@ -1,5 +1,6 @@
 import {
   AttributeValue,
+  BatchWriteItemCommand,
   DynamoDBClient,
   GetItemCommand,
   GetItemCommandInput,
@@ -210,6 +211,27 @@ export const inPlaceRecordUpdate = async (updatedRecord: TechRecordType<'get'>)
   }
 };
 
+// DO NOT USE THIS UNLESS FOR SEED DATA FOR PLATES
+export const insertBatchPlateSeedData = async (putRequests: any) => {
+  const command = new BatchWriteItemCommand({
+    RequestItems: {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+      [tableName]: putRequests,
+    },
+  });
+
+  try {
+    await ddbClient.send((command));
+  } catch (err) {
+    logger.error('Error in batch upload for plates: ', err);
+    // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+    throw new Error(
+      // eslint-disable-next-line max-len
+      'database client failed in batch plate uploader',
+    );
+  }
+};
+
 export const correctVrm = async (newRecord: TechRecordType<'get'>): Promise<object> => {
   logger.info('correcting a VRM');
   const putItemInput: PutItemCommandInput = {

src/util/generateBatchPlateData.ts

@@ -0,0 +1,222 @@
+/* eslint-disable no-param-reassign */
+import { TechRecordGETHGVSkeleton } from '@dvsa/cvs-type-definitions/types/v3/tech-record/get/hgv/skeleton';
+import { TechRecordGETTRLSkeleton } from '@dvsa/cvs-type-definitions/types/v3/tech-record/get/trl/skeleton';
+import { TechRecordGETTRLTestable } from '@dvsa/cvs-type-definitions/types/v3/tech-record/get/trl/testable';
+import { TechRecordGETHGV, TechRecordGETTRL } from '@dvsa/cvs-type-definitions/types/v3/tech-record/tech-record-verb-vehicle-type';
+import * as fs from 'fs';
+import completeHGVTechRecord from '../../tests/resources/techRecordCompleteHGVPlate.json'; // This record is labeled complete, and has all information for generating a plate. But ius actually marked as testable
+import completeTRLTechRecords from '../../tests/resources/technical-records-v3-no-plates.json';
+
+function padZeroes(number: string, size: number) {
+  number = number.toString();
+  while (number.length < size) number = `0${number}`;
+  return number;
+}
+
+export const generateBatchPlateData = () => {
+  let fileCount = 1;
+  const fileNames = [];
+  const date = new Date().toISOString();
+  let records = [];
+  const triggerData = [];
+  console.log('starting');
+  // complete TRLs
+  for (let i = 10; i <= 40000; i++) {
+    const record = JSON.parse(JSON.stringify(completeTRLTechRecords[0])) as TechRecordGETTRL;
+    const paddedIterator = padZeroes(i.toString(), 6);
+    record.systemNumber = `BPS${paddedIterator}`;
+    record.createdTimestamp = date;
+    record.vin = `BPV${paddedIterator}`;
+    record.partialVin = paddedIterator;
+    records.push(record);
+    triggerData.push({ systemNumber: record.systemNumber, createdTimestamp: record.createdTimestamp });
+
+    if (i % 1000 === 0) {
+      const fileName = `/tmp/technical-records-v3-with-batch-plates-${fileCount.toString()}.json`;
+      fileNames.push(fileName);
+      fs.writeFileSync(fileName, JSON.stringify(records, null, 2));
+      records = [];
+      fileCount++;
+    }
+  }
+
+  records = [];
+
+  // Complete HGVs
+  for (let i = 40001; i <= 80000; i++) {
+    const record = JSON.parse(JSON.stringify(completeHGVTechRecord)) as TechRecordGETHGV;
+    const paddedIterator = padZeroes(i.toString(), 6);
+    record.systemNumber = `BPS${paddedIterator}`;
+    record.createdTimestamp = date;
+    record.vin = `BPV${paddedIterator}`;
+    record.partialVin = paddedIterator;
+    record.primaryVrm = `${paddedIterator}Z`;
+
+    records.push(record);
+    triggerData.push({ systemNumber: record.systemNumber, createdTimestamp: record.createdTimestamp });
+
+    if (i % 1000 === 0) {
+      const fileName = `/tmp/technical-records-v3-with-batch-plates-${fileCount.toString()}.json`;
+      fileNames.push(fileName);
+      fs.writeFileSync(fileName, JSON.stringify(records, null, 2));
+      records = [];
+      fileCount++;
+    }
+  }
+
+  records = [];
+  // Missing functionCode
+  for (let i = 80001; i <= 80500; i++) {
+    const record = JSON.parse(JSON.stringify(completeTRLTechRecords[0])) as TechRecordGETTRL;
+    const paddedIterator = padZeroes(i.toString(), 6);
+    record.systemNumber = `BPS${paddedIterator}`;
+    record.createdTimestamp = date;
+    record.vin = `BPV${paddedIterator}`;
+    record.partialVin = paddedIterator;
+    delete record.techRecord_functionCode;
+    records.push(record);
+    triggerData.push({ systemNumber: record.systemNumber, createdTimestamp: record.createdTimestamp });
+  }
+
+  const fileName1 = `/tmp/technical-records-v3-with-batch-plates-${fileCount.toString()}.json`;
+  fileNames.push(fileName1);
+  fs.writeFileSync(fileName1, JSON.stringify(records, null, 2));
+  records = [];
+  fileCount++;
+
+  // missing model and functionCode
+  for (let i = 80501; i <= 90000; i++) {
+    const record = JSON.parse(JSON.stringify(completeTRLTechRecords[0])) as TechRecordGETTRL;
+    const paddedIterator = padZeroes(i.toString(), 6);
+    record.systemNumber = `BPS${paddedIterator}`;
+    record.createdTimestamp = date;
+    record.vin = `BPV${paddedIterator}`;
+    record.partialVin = paddedIterator;
+    delete record.techRecord_functionCode;
+    delete record.techRecord_model;
+    records.push(record);
+    triggerData.push({ systemNumber: record.systemNumber, createdTimestamp: record.createdTimestamp });
+
+    if (i % 1000 === 0) {
+      const fileName = `/tmp/technical-records-v3-with-batch-plates-${fileCount.toString()}.json`;
+      fileNames.push(fileName);
+      fs.writeFileSync(fileName, JSON.stringify(records, null, 2));
+      records = [];
+      fileCount++;
+    }
+  }
+
+  records = [];
+
+  // missing variantNumber
+  for (let i = 90001; i <= 98000; i++) {
+    const record = JSON.parse(JSON.stringify(completeHGVTechRecord)) as TechRecordGETHGV;
+    const paddedIterator = padZeroes(i.toString(), 6);
+    record.systemNumber = `BPS${paddedIterator}`;
+    record.createdTimestamp = date;
+    record.vin = `BPV${paddedIterator}`;
+    record.partialVin = paddedIterator;
+    record.primaryVrm = `${paddedIterator}Z`;
+    delete record.techRecord_variantNumber;
+
+    records.push(record);
+    triggerData.push({ systemNumber: record.systemNumber, createdTimestamp: record.createdTimestamp });
+
+    if (i % 1000 === 0) {
+      const fileName = `/tmp/technical-records-v3-with-batch-plates-${fileCount.toString()}.json`;
+      fileNames.push(fileName);
+      fs.writeFileSync(fileName, JSON.stringify(records, null, 2));
+      records = [];
+      fileCount++;
+    }
+  }
+  records = [];
+  console.log('98K');
+
+  // missing roadFriendly
+  for (let i = 98001; i <= 100000; i++) {
+    const record = JSON.parse(JSON.stringify(completeHGVTechRecord)) as TechRecordGETHGV;
+    const paddedIterator = padZeroes(i.toString(), 6);
+    record.systemNumber = `BPS${paddedIterator}`;
+    record.createdTimestamp = date;
+    record.vin = `BPV${paddedIterator}`;
+    record.partialVin = paddedIterator;
+    record.primaryVrm = `${paddedIterator}Z`;
+    delete record.techRecord_roadFriendly;
+
+    records.push(record);
+    triggerData.push({ systemNumber: record.systemNumber, createdTimestamp: record.createdTimestamp });
+  }
+  console.log('100K');
+  const fileName3 = `/tmp/technical-records-v3-with-batch-plates-${fileCount.toString()}.json`;
+  fileNames.push(fileName3);
+  fs.writeFileSync(fileName3, JSON.stringify(records, null, 2));
+  records = [];
+  fileCount++;
+
+  console.log('100K');
+  // HGV Skeleton
+  for (let i = 100001; i <= 101000; i++) {
+    const record = JSON.parse(JSON.stringify(completeHGVTechRecord)) as TechRecordGETHGVSkeleton;
+    const paddedIterator = padZeroes(i.toString(), 6);
+    record.systemNumber = `BPS${paddedIterator}`;
+    record.createdTimestamp = date;
+    record.vin = `BPV${paddedIterator}`;
+    record.partialVin = paddedIterator;
+    record.primaryVrm = `${paddedIterator}Z`;
+    record.techRecord_recordCompleteness = 'skeleton';
+
+    records.push(record);
+    triggerData.push({ systemNumber: record.systemNumber, createdTimestamp: record.createdTimestamp });
+  }
+  const fileName4 = `/tmp/technical-records-v3-with-batch-plates-${fileCount.toString()}.json`;
+  fileNames.push(fileName4);
+  fs.writeFileSync(fileName4, JSON.stringify(records, null, 2));
+  records = [];
+  fileCount++;
+  // TRL Skeleton
+  for (let i = 101001; i <= 102000; i++) {
+    const record = JSON.parse(JSON.stringify(completeTRLTechRecords[0])) as TechRecordGETTRLSkeleton;
+    const paddedIterator = padZeroes(i.toString(), 6);
+    record.systemNumber = `BPS${paddedIterator}`;
+    record.createdTimestamp = date;
+    record.vin = `BPV${paddedIterator}`;
+    record.partialVin = paddedIterator;
+    record.techRecord_recordCompleteness = 'skeleton';
+
+    records.push(record);
+    triggerData.push({ systemNumber: record.systemNumber, createdTimestamp: record.createdTimestamp });
+  }
+
+  const fileName5 = `/tmp/technical-records-v3-with-batch-plates-${fileCount.toString()}.json`;
+  fileNames.push(fileName5);
+  fs.writeFileSync(fileName5, JSON.stringify(records, null, 2));
+  records = [];
+  fileCount++;
+
+  // TRL Testable
+  for (let i = 102001; i <= 103000; i++) {
+    const record = JSON.parse(JSON.stringify(completeTRLTechRecords[0])) as TechRecordGETTRLTestable;
+    const paddedIterator = padZeroes(i.toString(), 6);
+    record.systemNumber = `BPS${paddedIterator}`;
+    record.createdTimestamp = date;
+    record.vin = `BPV${paddedIterator}`;
+    record.partialVin = paddedIterator;
+
+    records.push(record);
+    triggerData.push({ systemNumber: record.systemNumber, createdTimestamp: record.createdTimestamp });
+  }
+
+  const fileName6 = `/tmp/technical-records-v3-with-batch-plates-${fileCount.toString()}.json`;
+  fileNames.push(fileName6);
+  fs.writeFileSync(fileName6, JSON.stringify(records, null, 2));
+  records = [];
+  fileCount++;
+
+  console.log('written vehicles to seed data');
+  fs.writeFileSync('/tmp/trigger-data.json', JSON.stringify(triggerData, null, 2));
+  console.log('written vehicles to trigger data');
+
+  console.log(fileCount);
+  return fileNames;
+};

template.yml

@@ -192,6 +192,21 @@ Resources:
         Runtime: nodejs18.x
         Timeout: 20
 
+    UploadPlateSeed:
+      Type: 'AWS::Serverless::Function'
+      Properties:
+        CodeUri: src/handler/
+        Handler: uploadPlateSeedData.handler
+        Runtime: nodejs18.x
+        Timeout: 900
+        MemorySize: 10240
+        Events:
+          PlateSeed:
+            Type: Api
+            Properties:
+              Path: /v3/technical-records/uploadPlateSeed
+              Method: get
+
     LoadBatchPlate:
       Type: 'AWS::Serverless::Function'
       Properties:
@@ -203,11 +218,11 @@ Resources:
           S3Event:
             Type: S3
             Events: s3:ObjectCreated:*
-        Filter:
-          S3Key:
-            Rules:
-              - Name: prefix
-                Value: value
+      Filter:
+        S3Key:
+          Rules:
+            - Name: prefix
+              Value: value
 
     LocalQueue:
       Type: AWS::SQS::Queue