Skip to content

Commit

Permalink
Batching updates
Browse files Browse the repository at this point in the history 
 - Handle Promote and Copy batches in parallel
 - Adding in-progress statuses for not repeating the already initiated steps in schedulers and actions
 - Awaiting all copy and promote steps before updating the status to avoid impact of promote and copy in parallel
 - Cross refereincing & checking existence of same bathes in Copy from promote worker and vice versa before updating batch statuses as promoted
 - Checking image patterns and not process for GB styles if only image exists and no graybox styles exist
  • Loading branch information
@arshadparwaiz
arshadparwaiz committed Sep 9, 2024
1 parent e2cbd2c commit dab241b
Show file tree
Hide file tree
Showing 10 changed files with 307 additions and 133 deletions.
26 changes: 17 additions & 9 deletions actions/graybox/copy-sched.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,21 +44,29 @@ async function main(params) {

// Read the Batch Status in the current project's "batch_status.json" file
const batchStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/batch_status.json`);
logger.info(`In Copy Sched, Batch Status Json: ${JSON.stringify(batchStatusJson)}`);

const copyBatchesJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/copy_batches.json`);
// copy all params from json into the params object
const inputParams = projectStatusJson?.params;
Object.keys(inputParams).forEach((key) => {
params[key] = inputParams[key];
});

logger.info(`In Copy-sched copyBatchesJson: ${JSON.stringify(copyBatchesJson)}`);
// Find the first batch where status is 'processed'
logger.info(`In Copy-sched Copy Batches Json: ${JSON.stringify(copyBatchesJson)}`);
// Find the First Batch where status is 'processed', to promote one batch at a time
const batchEntry = Object.entries(copyBatchesJson)
.find(([batchName, copyBatchJson]) => copyBatchJson.status === 'processed');
// If no batch is found with status 'processed then nothing to promote', return
if (!batchEntry || !Array.isArray(batchEntry) || batchEntry.length === 0) {
responsePayload = 'No Copy Batches found with status "processed"';
return {
code: 200,
payload: responsePayload
};
}
const copyBatchName = batchEntry[0]; // Getting the key i.e. project path from the JSON entry, batchEntry[1] is the value

if (batchStatusJson[copyBatchName] === 'processed') {
if (batchStatusJson[copyBatchName] === 'processed' || batchStatusJson[copyBatchName] === 'promote_in_progress') {
// copy all params from json into the params object
const inputParams = projectStatusJson?.params;
Object.keys(inputParams).forEach((key) => {
params[key] = inputParams[key];
});
// Set the Project & Batch Name in params for the Copy Content Worker Action to read and process
params.project = project;
params.batchName = copyBatchName;
Expand Down
68 changes: 43 additions & 25 deletions actions/graybox/copy-worker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,27 +38,28 @@ async function main(params) {

logger.info('In Copy Worker, Processing Copy Content');

// Read the Batch Status in the current project's "batch_status.json" file
const batchStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`);

const promotedPathsJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/promoted_paths.json`) || {};

const promoteErrorsJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/promote_errors.json`);

const project = params.project || '';
const batchName = params.batchName || '';

// Combined existing If any promotes already exist in promoted_paths.json for the current batch either from Copy action or Promote Action
if (promotedPathsJson[batchName]) {
promotes = promotes.concat(promotedPathsJson[batchName]);
}
// Read the Batch Status in the current project's "batch_status.json" file
const batchStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/batch_status.json`);

const promoteErrorsJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/promote_errors.json`);

const copyBatchesJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/copy_batches.json`);

const copyBatchJson = copyBatchesJson[batchName] || {};

logger.info(`In Copy Worker, Copy File Paths: ${JSON.stringify(copyBatchJson)}`);

// Update & Write the Batch Status to in progress "batch_status.json" file
// So that the scheduler doesn't pick the same batch again
batchStatusJson[batchName] = 'copy_in_progress';
await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`, batchStatusJson);
// Write the copy batches JSON file
copyBatchesJson[batchName].status = 'promote_in_progress';
await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/copy_batches.json`, copyBatchesJson);

// Process the Copy Content
const copyFilePathsJson = copyBatchJson.files || [];
const copyPromises = copyFilePathsJson.map(async (copyPathsEntry) => {
Expand All @@ -79,9 +80,14 @@ async function main(params) {
// Wait for all copy operations to complete
await Promise.all(copyPromises);

logger.info(`In Copy Worker, Promotes: ${JSON.stringify(promotes)}`);
logger.info(`In Copy Worker, Promotes for batchname ${batchName} no.of files ${promotes.length}, files list: ${JSON.stringify(promotes)}`);
// Update the Promoted Paths in the current project's "promoted_paths.json" file
if (promotes.length > 0) {
const promotedPathsJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/promoted_paths.json`) || {};
// Combined existing If any promotes already exist in promoted_paths.json for the current batch either from Copy action or Promote Action
if (promotedPathsJson[batchName]) {
promotes = promotes.concat(promotedPathsJson[batchName]);
}
promotedPathsJson[batchName] = promotes;
await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/promoted_paths.json`, promotedPathsJson);
}
Expand All @@ -90,32 +96,44 @@ async function main(params) {
await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/promote_errors.json`, promoteErrorsJson.concat(failedPromotes));
}

// Update the Batch Status in the current project's "batch_status.json" file
if (batchStatusJson && batchStatusJson[batchName] && (promotes.length > 0 || failedPromotes.length > 0)) {
batchStatusJson[batchName] = 'promoted';
const promoteBatchesJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/promote_batches.json`);
const promoteBatchJson = promoteBatchesJson[batchName];
let markBatchAsPromoted = true;
if (promoteBatchJson) {
markBatchAsPromoted = promoteBatchJson.status === 'promoted';
}

// Write the updated batch_status.json file
await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`, batchStatusJson);
if (markBatchAsPromoted) {
// Update the Batch Status in the current project's "batch_status.json" file
if (batchStatusJson && batchStatusJson[batchName] && (promotes.length > 0 || failedPromotes.length > 0)) {
batchStatusJson[batchName] = 'promoted';
// Write the updated batch_status.json file
await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`, batchStatusJson);
}

// If all batches are promoted, then mark the project as 'promoted'
const allBatchesPromoted = Object.keys(batchStatusJson).every((key) => batchStatusJson[key] === 'promoted');
if (allBatchesPromoted) {
// Update the Project Status in JSON files
updateProjectStatus(gbRootFolder, experienceName, filesWrapper);
}
}

// Update the Copy Batch Status in the current project's "copy_batches.json" file
copyBatchesJson[batchName].status = 'promoted';
// Write the promote batches JSON file
// Write the copy batches JSON file
await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/copy_batches.json`, copyBatchesJson);

// Update the Project Excel with the Promote Status
try {
const sFailedPromoteStatuses = failedPromotes.length > 0 ? `Failed Promotes: \n${failedPromotes.join('\n')}` : '';
const promoteExcelValues = [[`Step 4 of 5: Copy Docx completed for Batch ${batchName}`, toUTCStr(new Date()), sFailedPromoteStatuses]];
const promoteExcelValues = [[`Step 4 of 5: Copy Content completed for Batch ${batchName}`, toUTCStr(new Date()), sFailedPromoteStatuses]];
await sharepoint.updateExcelTable(projectExcelPath, 'PROMOTE_STATUS', promoteExcelValues);
} catch (err) {
logger.error(`Error Occured while updating Excel during Graybox Promote Copy: ${err}`);
}

// Update the Project Status in JSON files
updateProjectStatus(gbRootFolder, experienceName, filesWrapper);

responsePayload = 'Copy Worker finished promoting content';
responsePayload = `Copy Worker finished promoting content for batch ${batchName}`;
logger.info(responsePayload);
return exitAction({
body: responsePayload,
Expand All @@ -131,20 +149,20 @@ async function main(params) {
* @returns updated project status
*/
async function updateProjectStatus(gbRootFolder, experienceName, filesWrapper) {
const projectQueue = await filesWrapper.readFileIntoObject('graybox_promote/project_queue.json');
const projectStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/status.json`);

// Update the Project Status in the current project's "status.json" file
projectStatusJson.status = 'promoted';
await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/status.json`, projectStatusJson);

// Update the Project Status in the parent "project_queue.json" file
const projectQueue = await filesWrapper.readFileIntoObject('graybox_promote/project_queue.json');
const index = projectQueue.findIndex((obj) => obj.projectPath === `${gbRootFolder}/${experienceName}`);
if (index !== -1) {
// Replace the object at the found index
projectQueue[index].status = 'promoted';
await filesWrapper.writeFile('graybox_promote/project_queue.json', projectQueue);
}
await filesWrapper.writeFile('graybox_promote/project_queue.json', projectQueue);
}

function exitAction(resp) {
Expand Down
16 changes: 12 additions & 4 deletions actions/graybox/initiate-promote-worker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ const BATCH_REQUEST_PREVIEW = 200;
// const BATCH_REQUEST_PREVIEW = 1; // TODO remove this line and uncomment the above line after testing

/**
* - Bulk Preview docx files
* - Bulk Preview Graybox files
* - GET markdown files using preview-url.md
* - Process markdown - process MDAST by cleaning it up
* - Generate updated Docx file using md2docx lib
Expand Down Expand Up @@ -76,13 +76,19 @@ async function main(params) {
// Promote Errors JSON
const promoteErrorsJson = [];

// Copy Batches JSON
const copyBatchesJson = {};

// Promote Batches JSON
const promoteBatchesJson = {};

// create batches to process the data
const gbFilesBatchArray = [];
const writeBatchJsonPromises = [];
for (let i = 0; i < gbFiles.length; i += BATCH_REQUEST_PREVIEW) {
for (let i = 0, batchCounter = 1; i < gbFiles.length; i += BATCH_REQUEST_PREVIEW, batchCounter += 1) {
const arrayChunk = gbFiles.slice(i, i + BATCH_REQUEST_PREVIEW);
gbFilesBatchArray.push(arrayChunk);
const batchName = `batch_${i + 1}`;
const batchName = `batch_${batchCounter}`;
batchStatusJson[`${batchName}`] = 'initiated';

// Each Files Batch is written to a batch_n.json file
Expand Down Expand Up @@ -145,6 +151,8 @@ async function main(params) {
await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/preview_errors.json`, projectPreviewErrorsJson);
await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/promoted_paths.json`, promotedPathsJson);
await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/promote_errors.json`, promoteErrorsJson);
await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/promote_batches.json`, promoteBatchesJson);
await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/copy_batches.json`, copyBatchesJson);

// read Graybox Project Json from AIO Files
const projectQueueJson = await filesWrapper.readFileIntoObject('graybox_promote/project_queue.json');
Expand All @@ -156,7 +164,7 @@ async function main(params) {

// process data in batches
let responsePayload;
responsePayload = 'Graybox Promote Worker action completed.';
responsePayload = 'Graybox Initiate Promote Worker action completed.';
logger.info(responsePayload);
return {
body: responsePayload,
Expand Down
23 changes: 20 additions & 3 deletions actions/graybox/preview-sched.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,33 @@ async function main(params) {
try {
const projectQueue = await filesWrapper.readFileIntoObject('graybox_promote/project_queue.json');
logger.info(`From Preview-sched Project Queue Json: ${JSON.stringify(projectQueue)}`);
if (!projectQueue) {
responsePayload = 'No projects in the queue';
logger.info(responsePayload);
return {
code: 200,
payload: responsePayload
};
}

// iterate the JSON array projects and extract the project_path where status is 'initiated'
const ongoingInitiatedProjectPaths = [];
const toBePreviewedProjectPaths = [];
projectQueue.forEach((project) => {
if (project.status === 'initiated' || project.status === 'promoted') {
ongoingInitiatedProjectPaths.push(project.projectPath);
toBePreviewedProjectPaths.push(project.projectPath);
}
});

ongoingInitiatedProjectPaths.forEach(async (project) => {
if (!toBePreviewedProjectPaths || toBePreviewedProjectPaths.length === 0) {
responsePayload = 'No projects in the queue with status initiated';
logger.info(responsePayload);
return {
code: 200,
payload: responsePayload
};
}

toBePreviewedProjectPaths.forEach(async (project) => {
const projectStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/status.json`);

// copy all params from json into the params object
Expand Down
Loading

0 comments on commit dab241b

Please sign in to comment.