From e768a6d1543fa4a43753f8c86ea7d7daa5916fc3 Mon Sep 17 00:00:00 2001 From: Arshad Mohammed <87503056+arshadparwaiz@users.noreply.github.com> Date: Wed, 7 Aug 2024 14:24:46 -0700 Subject: [PATCH] Promote API Batching Implementation --- actions/graybox/copy-content-worker.js | 103 ++++++++++ actions/graybox/copy-sched.js | 115 +++++++++++ actions/graybox/filesWrapper.js | 125 ++++++++++++ actions/graybox/preview-sched.js | 106 ++++++++++ actions/graybox/preview-worker.js | 191 ++++++++++++++++++ actions/graybox/process-docx-sched.js | 107 ++++++++++ actions/graybox/process-docx-worker.js | 230 ++++++++++++++++++++++ actions/graybox/promote-content-worker.js | 114 +++++++++++ actions/graybox/promote-sched.js | 113 +++++++++++ actions/graybox/promote-worker.js | 209 +++++++------------- actions/graybox/validateAction.js | 2 + app.config.yaml | 105 +++++++++- package-lock.json | 188 ++++++++++++++++-- package.json | 1 + 14 files changed, 1559 insertions(+), 150 deletions(-) create mode 100644 actions/graybox/copy-content-worker.js create mode 100644 actions/graybox/copy-sched.js create mode 100644 actions/graybox/filesWrapper.js create mode 100644 actions/graybox/preview-sched.js create mode 100644 actions/graybox/preview-worker.js create mode 100644 actions/graybox/process-docx-sched.js create mode 100644 actions/graybox/process-docx-worker.js create mode 100644 actions/graybox/promote-content-worker.js create mode 100644 actions/graybox/promote-sched.js diff --git a/actions/graybox/copy-content-worker.js b/actions/graybox/copy-content-worker.js new file mode 100644 index 0000000..f376f7d --- /dev/null +++ b/actions/graybox/copy-content-worker.js @@ -0,0 +1,103 @@ +/* ************************************************************************ +* ADOBE CONFIDENTIAL +* ___________________ +* +* Copyright 2024 Adobe +* All Rights Reserved. +* +* NOTICE: All information contained herein is, and remains +* the property of Adobe and its suppliers, if any. The intellectual +* and technical concepts contained herein are proprietary to Adobe +* and its suppliers and are protected by all applicable intellectual +* property laws, including trade secret and copyright laws. +* Dissemination of this information or reproduction of this material +* is strictly forbidden unless prior written permission is obtained +* from Adobe. +************************************************************************* */ + +const { + getAioLogger, handleExtension, toUTCStr +} = require('../utils'); +const AppConfig = require('../appConfig'); +const Sharepoint = require('../sharepoint'); +const initFilesWrapper = require('./filesWrapper'); + +const logger = getAioLogger(); + +async function main(params) { + logger.info('Graybox Promote Content Action triggered'); + + const appConfig = new AppConfig(params); + const { + spToken, adminPageUri, rootFolder, gbRootFolder, promoteIgnorePaths, experienceName, projectExcelPath, draftsOnly + } = appConfig.getPayload(); + + const sharepoint = new Sharepoint(appConfig); + + // process data in batches + const filesWrapper = await initFilesWrapper(logger); + let responsePayload; + const promotes = []; + const failedPromotes = []; + + logger.info('In Copy Content Worker, Processing Copy Content'); + + const project = params.project || ''; + const batchName = params.batchName || ''; + + const copyBatchesJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/copy_batches.json`); + + const copyFilePathsJson = copyBatchesJson[batchName] || {}; + + logger.info(`In Copy Content Worker, copyFilePaths: ${JSON.stringify(copyFilePathsJson)}`); + // Process the Copy Content + Object.entries(copyFilePathsJson).forEach(async ([copySourceFilePath, copyDestFilePath]) => { + // Download the grayboxed file and save it to default content location + const { fileDownloadUrl } = await sharepoint.getFileData(copySourceFilePath, true); + const file = await sharepoint.getFileUsingDownloadUrl(fileDownloadUrl); + const saveStatus = await sharepoint.saveFileSimple(file, copyDestFilePath); + + if (saveStatus?.success) { + promotes.push(copyDestFilePath); + } else if (saveStatus?.errorMsg?.includes('File is locked')) { + failedPromotes.push(`${copyDestFilePath} (locked file)`); + } else { + failedPromotes.push(copyDestFilePath); + } + }); + + responsePayload = 'Copy Content Worker finished promoting content'; + logger.info(responsePayload); + return exitAction({ + body: responsePayload, + statusCode: 200 + }); +} + +/** + * Update the Project Status in the current project's "status.json" file & the parent "ongoing_projects.json" file + * @param {*} gbRootFolder graybox root folder + * @param {*} experienceName graybox experience name + * @param {*} filesWrapper filesWrapper object + * @returns updated project status + */ +async function updateProjectStatus(gbRootFolder, experienceName, filesWrapper) { + const projects = await filesWrapper.readFileIntoObject('graybox_promote/ongoing_projects.json'); + const projectStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/status.json`); + + // Update the Project Status in the current project's "status.json" file + projectStatusJson.status = 'initial_preview_done'; + logger.info(`In Promote-content-worker After Processing Promote, Project Status Json: ${JSON.stringify(projectStatusJson)}`); + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/status.json`, projectStatusJson); + + // Update the Project Status in the parent "ongoing_projects.json" file + projects.find((p) => p.project_path === `${gbRootFolder}/${experienceName}`).status = 'initial_preview_done'; + logger.info(`In Promote-content-worker After Processing Promote, OnProjects Json: ${JSON.stringify(projects)}`); + await filesWrapper.writeFile('graybox_promote/ongoing_projects.json', projects); +} + +function exitAction(resp) { + return resp; +} + +exports.main = main; diff --git a/actions/graybox/copy-sched.js b/actions/graybox/copy-sched.js new file mode 100644 index 0000000..1433e67 --- /dev/null +++ b/actions/graybox/copy-sched.js @@ -0,0 +1,115 @@ +/* ************************************************************************ +* ADOBE CONFIDENTIAL +* ___________________ +* +* Copyright 2024 Adobe +* All Rights Reserved. +* +* NOTICE: All information contained herein is, and remains +* the property of Adobe and its suppliers, if any. The intellectual +* and technical concepts contained herein are proprietary to Adobe +* and its suppliers and are protected by all applicable intellectual +* property laws, including trade secret and copyright laws. +* Dissemination of this information or reproduction of this material +* is strictly forbidden unless prior written permission is obtained +* from Adobe. +************************************************************************* */ + +// eslint-disable-next-line import/no-extraneous-dependencies +const openwhisk = require('openwhisk'); +const { getAioLogger } = require('../utils'); +const { validateAction } = require('./validateAction'); +const AppConfig = require('../appConfig'); +const initFilesWrapper = require('./filesWrapper'); + +async function main(params) { + const logger = getAioLogger(); + const ow = openwhisk(); + let responsePayload = 'Graybox Copy Scheduler invoked'; + logger.info(responsePayload); + + const filesWrapper = await initFilesWrapper(logger); + + try { + const projects = await filesWrapper.readFileIntoObject('graybox_promote/ongoing_projects.json'); + logger.info(`From Copy-sched Ongoing Projects Json: ${JSON.stringify(projects)}`); + + // iterate the JSON array projects and extract the project_path where status is 'initiated' + const ongoingPorcessedProjects = []; + projects.forEach((project) => { + if (project.status === 'processed') { + ongoingPorcessedProjects.push(project.project_path); + } + }); + + ongoingPorcessedProjects.forEach(async (project) => { + const projectStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/status.json`); + logger.info(`In Copy-sched Projects Json: ${JSON.stringify(projectStatusJson)}`); + + const copyBatchesJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/copy_batches.json`); + // copy all params from json into the params object + const inputParams = projectStatusJson?.params; + Object.keys(inputParams).forEach((key) => { + params[key] = inputParams[key]; + }); + + Object.entries(copyBatchesJson).forEach(async ([batchName, copyFilePathsJson]) => { + // Set the Project & Batch Name in params for the Copy Content Worker Action to read and process + params.project = project; + params.batchName = batchName; + + try { + const appConfig = new AppConfig(params); + const grpIds = appConfig.getConfig().grayboxUserGroups; + const vActData = await validateAction(params, grpIds, params.ignoreUserCheck); + if (vActData && vActData.code !== 200) { + logger.info(`Validation failed: ${JSON.stringify(vActData)}`); + return vActData; + } + + return ow.actions.invoke({ + name: 'graybox/copy-content-worker', + blocking: false, + result: false, + params + }).then(async (result) => { + logger.info(result); + return { + code: 200, + payload: responsePayload + }; + }).catch(async (err) => { + responsePayload = 'Failed to invoke graybox copy action'; + logger.error(`${responsePayload}: ${err}`); + return { + code: 500, + payload: responsePayload + }; + }); + } catch (err) { + responsePayload = 'Unknown error occurred'; + logger.error(`${responsePayload}: ${err}`); + responsePayload = err; + } + + return { + code: 500, + payload: responsePayload, + }; + }); + }); + + logger.info(`Params length after: ${Object.keys(params).length}`); + } catch (err) { + responsePayload = 'Unknown error occurred'; + logger.error(`${responsePayload}: ${err}`); + responsePayload = err; + } + + return { + code: 500, + payload: responsePayload, + }; +} + +exports.main = main; diff --git a/actions/graybox/filesWrapper.js b/actions/graybox/filesWrapper.js new file mode 100644 index 0000000..4b42004 --- /dev/null +++ b/actions/graybox/filesWrapper.js @@ -0,0 +1,125 @@ +const Files = require('@adobe/aio-lib-files'); +const streamLib = require('stream'); + +const initFilesWrapper = async (logger) => { + const files = await Files.init(); + + const readFileInternal = async (filePath, logFileNotFound = true, options = {}) => { + try { + return await files.read(filePath, options); + } catch (err) { + if (logFileNotFound) { + logger.error(`Error while reading file ${filePath}: ${err.message}`); + } + return null; + } + }; + + const readFileIntoObject = async (filePath, logFileNotFound = true, options = {}) => { + const data = await readFileInternal(filePath, logFileNotFound, options); + try { + if (typeof input === "string") { + return JSON.parse(input); + } + return data ? JSON.parse(data.toString()) : {}; + } catch (err) { + if (logFileNotFound) { + logger.error(`Error while parsing file content of ${filePath}: ${err.message}`); + } + return {}; + } + }; + + const readProperties = async (filePath) => { + try { + return await files.getProperties(filePath); + } catch (err) { + logger.error(`Error while reading metadata of ${filePath}: ${err.message}`); + return null; + } + }; + + /** + * Return the file as Buffer or an empty Buffer, when reading the file errored out. + * + * @param filePath {string} path to the file to read + * @param logFileNotFound {boolean} whether a failure to read the file should be logged - defaults to true + * @param options {object} aio-lib-files "remoteReadOptions" - default to an empty object + * @returns {Buffer} the buffer with the file's content + */ + const readFileIntoBuffer = async (filePath, logFileNotFound = true, options = {}) => { + const data = await readFileInternal(filePath, logFileNotFound, options); + return data ?? Buffer.alloc(0); + }; + + const writeFile = async (filePath, content) => { + let finalData = content; + if (!Buffer.isBuffer(content) && typeof content !== 'string' && !(content instanceof String)) { + finalData = JSON.stringify(content); + } + try { + await files.write(filePath, finalData); + } catch (err) { + logger.error(`Error while writing file ${filePath}: ${err.message}`); + } + }; + + const createReadStream = async (filePath, options = {}) => files.createReadStream(filePath, options); + + const writeFileFromStream = async (filePath, stream) => { + try { + if (stream instanceof streamLib.Readable) { + const chunks = []; + // eslint-disable-next-line no-restricted-syntax + for await (const chunk of stream) { + chunks.push(chunk); + } + await files.write(filePath, Buffer.concat(chunks)); + const fileProps = await files.getProperties(filePath); + if (!fileProps || !fileProps?.contentLength) { + return 'Error: Failed to determine the file size of the stored document.'; + } + return null; + } + return 'Error: Unexpected stream.'; + } catch (err) { + return `Error while writing file ${filePath}: ${err.message}`; + } + }; + + const deleteObject = async (filePath) => { + try { + await files.delete(filePath); + } catch (err) { + logger.error(`Error while deleting ${filePath}: ${err.message}`); + } + }; + + const listFiles = async (filePath) => { + try { + return files.list(filePath); + } catch (err) { + logger.error(`Error while listing files: ${err.message}`); + return []; + } + }; + + const fileExists = async (filePath) => { + const fileList = await listFiles(filePath); + return !Array.isArray(fileList) || fileList.length !== 0; + }; + + return { + writeFileFromStream, + readFileIntoObject, + readProperties, + createReadStream, + listFiles, + fileExists, + writeFile, + deleteObject, + readFileIntoBuffer, + }; +}; + +module.exports = initFilesWrapper; diff --git a/actions/graybox/preview-sched.js b/actions/graybox/preview-sched.js new file mode 100644 index 0000000..cb82f8d --- /dev/null +++ b/actions/graybox/preview-sched.js @@ -0,0 +1,106 @@ +/* ************************************************************************ +* ADOBE CONFIDENTIAL +* ___________________ +* +* Copyright 2024 Adobe +* All Rights Reserved. +* +* NOTICE: All information contained herein is, and remains +* the property of Adobe and its suppliers, if any. The intellectual +* and technical concepts contained herein are proprietary to Adobe +* and its suppliers and are protected by all applicable intellectual +* property laws, including trade secret and copyright laws. +* Dissemination of this information or reproduction of this material +* is strictly forbidden unless prior written permission is obtained +* from Adobe. +************************************************************************* */ + +// eslint-disable-next-line import/no-extraneous-dependencies +const openwhisk = require('openwhisk'); +const { getAioLogger } = require('../utils'); +const { validateAction } = require('./validateAction'); +const AppConfig = require('../appConfig'); +const initFilesWrapper = require('./filesWrapper'); + +async function main(params) { + const logger = getAioLogger(); + const ow = openwhisk(); + let responsePayload = 'Graybox Preview Scheduler invoked'; + logger.info(responsePayload); + + const filesWrapper = await initFilesWrapper(logger); + + try { + const projects = await filesWrapper.readFileIntoObject('graybox_promote/ongoing_projects.json'); + logger.info(`From Preview-sched Ongoing Projects Json: ${JSON.stringify(projects)}`); + + // iterate the JSON array projects and extract the project_path where status is 'initiated' + const ongoingInitiatedProjects = []; + projects.forEach((project) => { + if (project.status === 'initiated') { + ongoingInitiatedProjects.push(project.project_path); + } + }); + + ongoingInitiatedProjects.forEach(async (project) => { + const projectStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/status.json`); + logger.info(`In Preview-sched Projects Json: ${JSON.stringify(projectStatusJson)}`); + + // copy all params from json into the params object + const inputParams = projectStatusJson?.params; + Object.keys(inputParams).forEach((key) => { + params[key] = inputParams[key]; + }); + + try { + const appConfig = new AppConfig(params); + const grpIds = appConfig.getConfig().grayboxUserGroups; + const vActData = await validateAction(params, grpIds, params.ignoreUserCheck); + if (vActData && vActData.code !== 200) { + logger.info(`Validation failed: ${JSON.stringify(vActData)}`); + return vActData; + } + + return ow.actions.invoke({ + name: 'graybox/preview-worker', + blocking: false, + result: false, + params + }).then(async (result) => { + logger.info(result); + return { + code: 200, + payload: responsePayload + }; + }).catch(async (err) => { + responsePayload = 'Failed to invoke graybox preview action'; + logger.error(`${responsePayload}: ${err}`); + return { + code: 500, + payload: responsePayload + }; + }); + } catch (err) { + responsePayload = 'Unknown error occurred'; + logger.error(`${responsePayload}: ${err}`); + responsePayload = err; + } + + return { + code: 500, + payload: responsePayload, + }; + }); + } catch (err) { + responsePayload = 'Unknown error occurred'; + logger.error(`${responsePayload}: ${err}`); + responsePayload = err; + } + + return { + code: 500, + payload: responsePayload, + }; +} + +exports.main = main; diff --git a/actions/graybox/preview-worker.js b/actions/graybox/preview-worker.js new file mode 100644 index 0000000..563eb99 --- /dev/null +++ b/actions/graybox/preview-worker.js @@ -0,0 +1,191 @@ +/* ************************************************************************ +* ADOBE CONFIDENTIAL +* ___________________ +* +* Copyright 2024 Adobe +* All Rights Reserved. +* +* NOTICE: All information contained herein is, and remains +* the property of Adobe and its suppliers, if any. The intellectual +* and technical concepts contained herein are proprietary to Adobe +* and its suppliers and are protected by all applicable intellectual +* property laws, including trade secret and copyright laws. +* Dissemination of this information or reproduction of this material +* is strictly forbidden unless prior written permission is obtained +* from Adobe. +************************************************************************* */ + +const { + getAioLogger, handleExtension, toUTCStr +} = require('../utils'); +const AppConfig = require('../appConfig'); +const HelixUtils = require('../helixUtils'); +const Sharepoint = require('../sharepoint'); +const initFilesWrapper = require('./filesWrapper'); + +const logger = getAioLogger(); + +async function main(params) { + logger.info('Graybox Preview Action triggered'); + + const appConfig = new AppConfig(params); + const { + spToken, adminPageUri, rootFolder, gbRootFolder, promoteIgnorePaths, experienceName, projectExcelPath, draftsOnly + } = appConfig.getPayload(); + + const sharepoint = new Sharepoint(appConfig); + + // process data in batches + const helixUtils = new HelixUtils(appConfig); + // Batch Name to Array of Batch Preview Statuses mapping + const previewStatuses = {}; + const filesWrapper = await initFilesWrapper(logger); + let responsePayload; + if (helixUtils.canBulkPreview(true)) { + logger.info('In Preview Worker, Bulk Previewing Graybox files'); + + const batchStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`); + + logger.info(`In Preview-Worker, batchStatusJson: ${JSON.stringify(batchStatusJson)}`); + + const noofbatches = batchStatusJson !== undefined ? Object.keys(batchStatusJson).length : 0; + // iterate over batch_status.json file and process each batch + const batchResults = {}; + + // Read the Batch JSON file into an array + const i = 0; // Start with counter as 0 + await iterateAndReadBatchJson(i, batchResults, noofbatches, batchStatusJson); + + // Write the updated batch_status.json file + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`, batchStatusJson); + logger.info(`Updated Batch Status Json: ${JSON.stringify(batchStatusJson)}`); + + // PreviewStatuses is an object with keys(batchNames) mapping to arrays(previewStauses) + const failedPreviews = Object.keys(previewStatuses).reduce((acc, key) => { + const filteredStatuses = previewStatuses[key] + .filter((status) => !status.success) // Filter out failed statuses + .map((status) => status.path); // Map to get the path of the failed status + return acc.concat(filteredStatuses); // Concatenate to the accumulator + }, []); + // Now failedPreviews contains all the paths from the filtered and mapped arrays + + const previewStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/preview_status.json`); + const previewErrorsJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/preview_errors.json`); + + // Combine the Preview Statuses for each batch read from AIO Json with the Preview Statuses + previewStatusJson.forEach((batchName) => { + const batchPreviewStatuses = previewStatuses[batchName]; + if (batchPreviewStatuses) { + previewStatuses[batchName] = batchPreviewStatuses.concat(previewStatusJson[batchName]); + } + }); + + // Write the updated preview_errors.json file + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/preview_status.json`, previewStatuses); + + // Write the updated preview_errors.json file + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/preview_errors.json`, previewErrorsJson.concat(failedPreviews)); + + const updatedBatchStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`); + logger.info(`Updated Project Batch Status Json: ${JSON.stringify(updatedBatchStatusJson)}`); + + const updatedPreviewStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/preview_status.json`); + logger.info(`Updated Project Preview Status Json: ${JSON.stringify(updatedPreviewStatusJson)}`); + + const updatedPreviewErrorsJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/preview_errors.json`); + logger.info(`Updated Project Preview Errors Json: ${JSON.stringify(updatedPreviewErrorsJson)}`); + + // Update the Project Status in the current project's "status.json" file & the parent "ongoing_projects.json" file + await updateProjectStatus(gbRootFolder, experienceName, filesWrapper); + + try { + logger.info('Updating project excel file with status'); + const sFailedPreviews = failedPreviews.length > 0 ? `Failed Previews(Promote won't happen for these): \n${failedPreviews.join('\n')}` : ''; + const excelValues = [['Step 1 of 4: Initial Preview of Graybox completed', toUTCStr(new Date()), sFailedPreviews]]; + // Update Preview Status + await sharepoint.updateExcelTable(projectExcelPath, 'PROMOTE_STATUS', excelValues); + } catch (err) { + logger.error('Error Occured while updating Excel during Graybox Initial Preview'); + } + + responsePayload = 'Graybox Preview Worker action completed.'; + } else { + responsePayload = 'Bulk Preview not enabled for Graybox Content Tree'; + } + logger.info(responsePayload); + return exitAction({ + body: responsePayload, + statusCode: 200 + }); + + /** + * Iterate over the Batch JSON files, read those into an array and perform Bulk Preview + * @param {*} i counter + * @param {*} batchResults batchResults array + * @param {*} noofbatches total no of batches + * @param {*} filesWrapper filesWrapper object + * @param {*} gbRootFolder graybox root folder + * @param {*} experienceName graybox experience name + */ + async function iterateAndReadBatchJson(i, batchResults, noofbatches, batchStatusJson) { + const batchName = `batch_${i + 1}`; + + if (i < noofbatches && batchStatusJson[batchName] === 'initiated') { + // Read the Batch JSON file into an batchResults JSON object + const batchJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/batches/${batchName}.json`); + batchResults[`${batchName}`] = batchJson; + + // Perform Bulk Preview of a Batch of Graybox files + await previewBatch(batchName, batchResults, batchStatusJson); + + if (i + 1 < noofbatches) { // Recrusive call next batch only if batch exists + // Recursively call the function to process the next batch + await iterateAndReadBatchJson(i + 1, batchResults, noofbatches, batchStatusJson); + } + } + } + + /** + * Perform a Bulk Preview on a Batch of Graybox files + * @param {*} batchName batchResults array + * @param {*} previewStatuses returned preview statuses + * @param {*} helixUtils helixUtils object + * @param {*} experienceName graybox experience name + */ + async function previewBatch(batchName, batchResults, batchStatusJson) { + const batchJson = batchResults[batchName]; + const paths = []; + batchJson.forEach((gbFile) => paths.push(handleExtension(gbFile.filePath))); + // Perform Bulk Preview of a Batch of Graybox files + previewStatuses[batchName] = await helixUtils.bulkPreview(paths, helixUtils.getOperations().PREVIEW, experienceName, true); + batchStatusJson[batchName] = 'initial_preview_done'; + } +} + +/** + * Update the Project Status in the current project's "status.json" file & the parent "ongoing_projects.json" file + * @param {*} gbRootFolder graybox root folder + * @param {*} experienceName graybox experience name + * @param {*} filesWrapper filesWrapper object + * @returns updated project status + */ +async function updateProjectStatus(gbRootFolder, experienceName, filesWrapper) { + const projects = await filesWrapper.readFileIntoObject('graybox_promote/ongoing_projects.json'); + const projectStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/status.json`); + + // Update the Project Status in the current project's "status.json" file + projectStatusJson.status = 'initial_preview_done'; + logger.info(`In Preview-sched After Processing Preview, Project Status Json: ${JSON.stringify(projectStatusJson)}`); + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/status.json`, projectStatusJson); + + // Update the Project Status in the parent "ongoing_projects.json" file + projects.find((p) => p.project_path === `${gbRootFolder}/${experienceName}`).status = 'initial_preview_done'; + logger.info(`In Preview-sched After Processing Preview, OnProjects Json: ${JSON.stringify(projects)}`); + await filesWrapper.writeFile('graybox_promote/ongoing_projects.json', projects); +} + +function exitAction(resp) { + return resp; +} + +exports.main = main; diff --git a/actions/graybox/process-docx-sched.js b/actions/graybox/process-docx-sched.js new file mode 100644 index 0000000..8a69e5d --- /dev/null +++ b/actions/graybox/process-docx-sched.js @@ -0,0 +1,107 @@ +/* ************************************************************************ +* ADOBE CONFIDENTIAL +* ___________________ +* +* Copyright 2024 Adobe +* All Rights Reserved. +* +* NOTICE: All information contained herein is, and remains +* the property of Adobe and its suppliers, if any. The intellectual +* and technical concepts contained herein are proprietary to Adobe +* and its suppliers and are protected by all applicable intellectual +* property laws, including trade secret and copyright laws. +* Dissemination of this information or reproduction of this material +* is strictly forbidden unless prior written permission is obtained +* from Adobe. +************************************************************************* */ + +// eslint-disable-next-line import/no-extraneous-dependencies +const openwhisk = require('openwhisk'); +const { getAioLogger } = require('../utils'); +const { validateAction } = require('./validateAction'); +const AppConfig = require('../appConfig'); +const initFilesWrapper = require('./filesWrapper'); + +async function main(params) { + const logger = getAioLogger(); + const ow = openwhisk(); + let responsePayload = 'Graybox Process Docx Scheduler invoked'; + logger.info(responsePayload); + + const filesWrapper = await initFilesWrapper(logger); + + try { + const projects = await filesWrapper.readFileIntoObject('graybox_promote/ongoing_projects.json'); + logger.info(`In Process-docx-sched Ongoing Projects Json: ${JSON.stringify(projects)}`); + + // iterate the JSON array projects and extract the project_path where status is 'initial_preview_done' + const ongoingPreviewedProjects = []; + projects.forEach((project) => { + if (project.status === 'initial_preview_done') { + ongoingPreviewedProjects.push(project.project_path); + } + }); + + ongoingPreviewedProjects.forEach(async (project) => { + logger.info(`Project Path: ${project}`); + const projectStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/status.json`); + logger.info(`In Process-docx-sched Projects Json: ${JSON.stringify(projectStatusJson)}`); + + // copy all params from json into the params object + const inputParams = projectStatusJson?.params; + Object.keys(inputParams).forEach((key) => { + params[key] = inputParams[key]; + }); + + try { + const appConfig = new AppConfig(params); + const grpIds = appConfig.getConfig().grayboxUserGroups; + const vActData = await validateAction(params, grpIds, params.ignoreUserCheck); + if (vActData && vActData.code !== 200) { + logger.info(`Validation failed: ${JSON.stringify(vActData)}`); + return vActData; + } + + return ow.actions.invoke({ + name: 'graybox/process-docx-worker', + blocking: false, + result: false, + params + }).then(async (result) => { + logger.info(result); + return { + code: 200, + payload: responsePayload + }; + }).catch(async (err) => { + responsePayload = 'Failed to invoke graybox process docx action'; + logger.error(`${responsePayload}: ${err}`); + return { + code: 500, + payload: responsePayload + }; + }); + } catch (err) { + responsePayload = 'Unknown error occurred'; + logger.error(`${responsePayload}: ${err}`); + responsePayload = err; + } + + return { + code: 500, + payload: responsePayload, + }; + }); + } catch (err) { + responsePayload = 'Unknown error occurred'; + logger.error(`${responsePayload}: ${err}`); + responsePayload = err; + } + + return { + code: 500, + payload: responsePayload, + }; +} + +exports.main = main; diff --git a/actions/graybox/process-docx-worker.js b/actions/graybox/process-docx-worker.js new file mode 100644 index 0000000..5a33381 --- /dev/null +++ b/actions/graybox/process-docx-worker.js @@ -0,0 +1,230 @@ +/* ************************************************************************ +* ADOBE CONFIDENTIAL +* ___________________ +* +* Copyright 2024 Adobe +* All Rights Reserved. +* +* NOTICE: All information contained herein is, and remains +* the property of Adobe and its suppliers, if any. The intellectual +* and technical concepts contained herein are proprietary to Adobe +* and its suppliers and are protected by all applicable intellectual +* property laws, including trade secret and copyright laws. +* Dissemination of this information or reproduction of this material +* is strictly forbidden unless prior written permission is obtained +* from Adobe. +************************************************************************* */ + +const fetch = require('node-fetch'); +const { + getAioLogger, handleExtension, toUTCStr +} = require('../utils'); +const AppConfig = require('../appConfig'); +const HelixUtils = require('../helixUtils'); +const Sharepoint = require('../sharepoint'); +const updateDocument = require('../docxUpdater'); +const initFilesWrapper = require('./filesWrapper'); + +const gbStyleExpression = 'gb-'; // graybox style expression. need to revisit if there are any more styles to be considered. +const gbDomainSuffix = '-graybox'; + +const BATCH_REQUEST_PROMOTE = 200; + +const logger = getAioLogger(); + +async function main(params) { + logger.info('Graybox Process Docx Action triggered'); + + const appConfig = new AppConfig(params); + const { + spToken, adminPageUri, rootFolder, gbRootFolder, promoteIgnorePaths, experienceName, projectExcelPath, draftsOnly + } = appConfig.getPayload(); + + const sharepoint = new Sharepoint(appConfig); + // process data in batches + const helixUtils = new HelixUtils(appConfig); + const filesWrapper = await initFilesWrapper(logger); + let responsePayload; + + // Get the Helix Admin API Key for the Graybox content tree, needed for accessing (with auth) Images in graybox tree + const helixAdminApiKey = helixUtils.getAdminApiKey(true); + + const previewStatuses = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/preview_status.json`); + + logger.info(`In Process-doc-worker, previewStatuses: ${JSON.stringify(previewStatuses)}`); + if (!previewStatuses) { + responsePayload = 'No preview statuses found'; + logger.info(responsePayload); + return exitAction({ + body: responsePayload, + statusCode: 200 + }); + } + const processFilesParams = { + previewStatuses, + experienceName, + helixAdminApiKey, + sharepoint, + helixUtils, + appConfig, + filesWrapper, + gbRootFolder + }; + // Promote Graybox files to the default content tree + const { promotes, failedPromotes } = await processFiles(processFilesParams); + + // Update Promote Status + const sFailedPromoteStatuses = failedPromotes.length > 0 ? `Failed Promotes: \n${failedPromotes.join('\n')}` : ''; + const promoteExcelValues = [['Promote completed', toUTCStr(new Date()), sFailedPromoteStatuses]]; + await sharepoint.updateExcelTable(projectExcelPath, 'PROMOTE_STATUS', promoteExcelValues); + + responsePayload = 'Processing of Graybox Content Tree completed'; + logger.info(responsePayload); + return exitAction({ + body: responsePayload, + statusCode: 200 + }); +} + +/** +* Process files to clean up GB Styles and Link +* @returns +*/ +async function processFiles({ + previewStatuses, experienceName, helixAdminApiKey, sharepoint, helixUtils, appConfig, filesWrapper, gbRootFolder +}) { + const promotes = []; + const failedPromotes = []; + const options = {}; + // Passing isGraybox param true to fetch graybox Hlx Admin API Key + const grayboxHlxAdminApiKey = helixUtils.getAdminApiKey(true); + if (grayboxHlxAdminApiKey) { + options.headers = new fetch.Headers(); + options.headers.append('Authorization', `token ${grayboxHlxAdminApiKey}`); + } + + // Read the Ongoing Projects JSON file + const projects = await filesWrapper.readFileIntoObject('graybox_promote/ongoing_projects.json'); + + // Read the Project Status in the current project's "status.json" file + const projectStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/status.json`); + + // Read the Batch Status in the current project's "batch_status.json" file + const batchStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`); + + logger.info(`In Process-doc-worker, batchStatusJson: ${JSON.stringify(batchStatusJson)}`); + const promoteBatchesJson = {}; + const copyBatchesJson = {}; + let promoteBatchCount = 0; + let copyBatchCount = 0; + + // iterate through preview statuses, generate docx files and create promote & copy batches + Object.keys(previewStatuses).forEach(async (batchName) => { + logger.info(`In Process-doc-worker Processing batch ${batchName}`); + const batchPreviewStatuses = previewStatuses[batchName]; + logger.info(`In Process-doc-worker previewStatuses[batchName] ${JSON.stringify(previewStatuses[batchName])} batch ${batchName} with ${batchPreviewStatuses.length} files`); + logger.info(`In Process-doc-worker batchStatusJson[batchName] ${batchStatusJson[batchName]}`); + logger.info(`In Process-doc-worker batchStatusJson[batchName] === 'initial_preview_done' ${batchStatusJson[batchName] === 'initial_preview_done'}`); + + // Check if Step 2 finished, do the Step 3, if the batch status is 'initial_preview_done' then process the batch + if (batchStatusJson[batchName] === 'initial_preview_done') { + logger.info(`In Process-doc-worker batchStatusJson[batchName] ${batchStatusJson[batchName]}`); + + const allPromises = batchPreviewStatuses.map(async (status) => { + if (status.success && status.mdPath) { // If the file is successfully initial previewed and has a mdPath then process the file + const response = await sharepoint.fetchWithRetry(`${status.mdPath}`, options); + const content = await response.text(); + let docx; + const sp = await appConfig.getSpConfig(); + + if (content.includes(experienceName) || content.includes(gbStyleExpression) || content.includes(gbDomainSuffix)) { + // Process the Graybox Styles and Links with Mdast to Docx conversion + docx = await updateDocument(content, experienceName, helixAdminApiKey); + if (docx) { + const destinationFilePath = `${status.path.substring(0, status.path.lastIndexOf('/') + 1).replace('/'.concat(experienceName), '')}${status.fileName}`; + // Write the processed documents to the AIO folder for docx files + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/docx${destinationFilePath}`, docx); + // Create Promote Batches + const promoteBatchName = `batch_${promoteBatchCount + 1}`; + logger.info(`In Process-doc-worker Promote Batch Name: ${promoteBatchName}`); + const promoteBatchJson = promoteBatchesJson[promoteBatchName]; + logger.info(`In Process-doc-worker Promote Batch JSON: ${JSON.stringify(promoteBatchJson)}`); + if (!promoteBatchJson) { + promoteBatchesJson[promoteBatchName] = []; + } + promoteBatchesJson[promoteBatchName].push(destinationFilePath); + logger.info(`In Process-doc-worker Promote Batch JSON after push: ${JSON.stringify(promoteBatchesJson)}`); + + // If the promote batch count reaches the limit, increment the promote batch count + if (promoteBatchCount === BATCH_REQUEST_PROMOTE) { + promoteBatchCount += 1; + } + // Write the promote batches JSON file + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/promote_batches.json`, promoteBatchesJson); + batchStatusJson[batchName] = 'processed'; + } + } else { + // Copy Source full path with file name and extension + const copySourceFilePath = `${status.path.substring(0, status.path.lastIndexOf('/') + 1)}${status.fileName}`; + // Copy Destination folder path, no file name + const copyDestinationFolder = `${status.path.substring(0, status.path.lastIndexOf('/')).replace('/'.concat(experienceName), '')}`; + const copyDestFilePath = `${copyDestinationFolder}/${status.fileName}`; + + // Create Copy Batches + const copyBatchName = `batch_${copyBatchCount + 1}`; + let copyBatchJson = copyBatchesJson[copyBatchName]; + if (!copyBatchJson) { + copyBatchJson = {}; + } + copyBatchJson[copySourceFilePath] = copyDestFilePath; + copyBatchesJson[copyBatchName] = copyBatchJson; + + // If the copy batch count reaches the limit, increment the copy batch count + if (copyBatchCount === BATCH_REQUEST_PROMOTE) { + copyBatchCount += 1; // Increment the copy batch count + } + logger.info(`In Process-doc-worker Copy Batch JSON after push: ${JSON.stringify(copyBatchesJson)}`); + // Write the copy batches JSON file + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/copy_batches.json`, copyBatchesJson); + + batchStatusJson[batchName] = 'processed'; + } + } + }); + await Promise.all(allPromises); // await all async functions in the array are executed, before updating the status in the graybox project excel + + // Update each Batch Status in the current project's "batch_status.json" file + batchStatusJson[batchName] = 'processed'; + } + }); + + // Update the Project Status in the current project's "status.json" file & the parent "ongoing_projects.json" file + updateProjectStatus(batchStatusJson, projectStatusJson, gbRootFolder, experienceName, filesWrapper); + return { promotes, failedPromotes }; +} + +/** + * Update the Project Status in the current project's "status.json" file & the parent "ongoing_projects.json" file + * @param {*} gbRootFolder graybox root folder + * @param {*} experienceName graybox experience name + * @param {*} filesWrapper filesWrapper object + * @returns updated project status + */ +async function updateProjectStatus(batchStatusJson, projectStatusJson, gbRootFolder, experienceName, filesWrapper) { + const projects = await filesWrapper.readFileIntoObject('graybox_promote/ongoing_projects.json'); + // Write the Project Status in the current project's "status.json" file + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/status.json`, projectStatusJson); + // Update the Project Status & Batch Status in the current project's "status.json" & updated batch_status.json file respectively + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`, batchStatusJson); + + // Update the Project Status in the parent "ongoing_projects.json" file + projects.find((p) => p.project_path === `${gbRootFolder}/${experienceName}`).status = 'processed'; + logger.info(`In Process-docx-worker After Processing Docx, OnProjects Json: ${JSON.stringify(projects)}`); + await filesWrapper.writeFile('graybox_promote/ongoing_projects.json', projects); +} + +function exitAction(resp) { + return resp; +} + +exports.main = main; diff --git a/actions/graybox/promote-content-worker.js b/actions/graybox/promote-content-worker.js new file mode 100644 index 0000000..b110971 --- /dev/null +++ b/actions/graybox/promote-content-worker.js @@ -0,0 +1,114 @@ +/* ************************************************************************ +* ADOBE CONFIDENTIAL +* ___________________ +* +* Copyright 2024 Adobe +* All Rights Reserved. +* +* NOTICE: All information contained herein is, and remains +* the property of Adobe and its suppliers, if any. The intellectual +* and technical concepts contained herein are proprietary to Adobe +* and its suppliers and are protected by all applicable intellectual +* property laws, including trade secret and copyright laws. +* Dissemination of this information or reproduction of this material +* is strictly forbidden unless prior written permission is obtained +* from Adobe. +************************************************************************* */ + +const { + getAioLogger, handleExtension, toUTCStr +} = require('../utils'); +const AppConfig = require('../appConfig'); +const HelixUtils = require('../helixUtils'); +const Sharepoint = require('../sharepoint'); +const initFilesWrapper = require('./filesWrapper'); + +const logger = getAioLogger(); + +async function main(params) { + logger.info('Graybox Promote Content Action triggered'); + + const appConfig = new AppConfig(params); + const { + spToken, adminPageUri, rootFolder, gbRootFolder, promoteIgnorePaths, experienceName, projectExcelPath, draftsOnly + } = appConfig.getPayload(); + + const sharepoint = new Sharepoint(appConfig); + + // process data in batches + const helixUtils = new HelixUtils(appConfig); + const filesWrapper = await initFilesWrapper(logger); + let responsePayload; + const promotes = []; + const failedPromotes = []; + + logger.info('In Promote Content Worker, Processing Promote Content'); + + // const promoteFilePaths = params.promoteFilePaths || []; + + const project = params.project || ''; + const batchName = params.batchName || ''; + + const promoteBatchesJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/promote_batches.json`); + logger.info(`In Promote-sched Promote Batches Json: ${JSON.stringify(promoteBatchesJson)}`); + + const promoteFilePaths = promoteBatchesJson[batchName] || []; + + logger.info(`In Promote Content Worker, promoteFilePaths: ${JSON.stringify(promoteFilePaths)}`); + // Process the Promote Content + promoteFilePaths.forEach(async (promoteFilePath) => { + const promoteDocx = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/docx${promoteFilePath}`); + if (promoteDocx) { + logger.info('in promoteDocx'); + logger.info(`In Promote Content Worker, Promote Docx: ${JSON.stringify(promoteDocx)}`); + } + const saveStatus = await sharepoint.saveFileSimple(promoteDocx, promoteFilePath); + logger.info(`In Promote Content Worker, Save Status of ${promoteFilePath}: ${JSON.stringify(saveStatus)}`); + + if (saveStatus?.success) { + promotes.push(promoteFilePath); + } else if (saveStatus?.errorMsg?.includes('File is locked')) { + failedPromotes.push(`${promoteFilePath} (locked file)`); + } else { + failedPromotes.push(promoteFilePath); + } + }); + + logger.info(`In Promote Content Worker, Promotes: ${JSON.stringify(promotes)}`); + logger.info(`In Promote Content Worker, Failed Promotes: ${JSON.stringify(failedPromotes)}`); + + responsePayload = 'Promote Content Worker finished promoting content'; + logger.info(responsePayload); + return exitAction({ + body: responsePayload, + statusCode: 200 + }); +} + +/** + * Update the Project Status in the current project's "status.json" file & the parent "ongoing_projects.json" file + * @param {*} gbRootFolder graybox root folder + * @param {*} experienceName graybox experience name + * @param {*} filesWrapper filesWrapper object + * @returns updated project status + */ +async function updateProjectStatus(gbRootFolder, experienceName, filesWrapper) { + const projects = await filesWrapper.readFileIntoObject('graybox_promote/ongoing_projects.json'); + const projectStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/status.json`); + + // Update the Project Status in the current project's "status.json" file + projectStatusJson.status = 'initial_preview_done'; + logger.info(`In Promote-content-worker After Processing Promote, Project Status Json: ${JSON.stringify(projectStatusJson)}`); + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/status.json`, projectStatusJson); + + // Update the Project Status in the parent "ongoing_projects.json" file + projects.find((p) => p.project_path === `${gbRootFolder}/${experienceName}`).status = 'initial_preview_done'; + logger.info(`In Promote-content-worker After Processing Promote, OnProjects Json: ${JSON.stringify(projects)}`); + await filesWrapper.writeFile('graybox_promote/ongoing_projects.json', projects); +} + +function exitAction(resp) { + return resp; +} + +exports.main = main; diff --git a/actions/graybox/promote-sched.js b/actions/graybox/promote-sched.js new file mode 100644 index 0000000..adc1ae0 --- /dev/null +++ b/actions/graybox/promote-sched.js @@ -0,0 +1,113 @@ +/* ************************************************************************ +* ADOBE CONFIDENTIAL +* ___________________ +* +* Copyright 2024 Adobe +* All Rights Reserved. +* +* NOTICE: All information contained herein is, and remains +* the property of Adobe and its suppliers, if any. The intellectual +* and technical concepts contained herein are proprietary to Adobe +* and its suppliers and are protected by all applicable intellectual +* property laws, including trade secret and copyright laws. +* Dissemination of this information or reproduction of this material +* is strictly forbidden unless prior written permission is obtained +* from Adobe. +************************************************************************* */ + +// eslint-disable-next-line import/no-extraneous-dependencies +const openwhisk = require('openwhisk'); +const { getAioLogger } = require('../utils'); +const { validateAction } = require('./validateAction'); +const AppConfig = require('../appConfig'); +const initFilesWrapper = require('./filesWrapper'); + +async function main(params) { + const logger = getAioLogger(); + const ow = openwhisk(); + let responsePayload = 'Graybox Promote Scheduler invoked'; + logger.info(responsePayload); + + const filesWrapper = await initFilesWrapper(logger); + + try { + const projects = await filesWrapper.readFileIntoObject('graybox_promote/ongoing_projects.json'); + logger.info(`From Promote-sched Ongoing Projects Json: ${JSON.stringify(projects)}`); + + // iterate the JSON array projects and extract the project_path where status is 'initiated' + const ongoingPorcessedProjects = []; + projects.forEach((project) => { + if (project.status === 'processed') { + ongoingPorcessedProjects.push(project.project_path); + } + }); + + ongoingPorcessedProjects.forEach(async (project) => { + const projectStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/status.json`); + + const promoteBatchesJson = await filesWrapper.readFileIntoObject(`graybox_promote${project}/promote_batches.json`); + + // copy all params from json into the params object + const inputParams = projectStatusJson?.params; + Object.keys(inputParams).forEach((key) => { + params[key] = inputParams[key]; + }); + + Object.entries(promoteBatchesJson).forEach(async ([batchName, promoteFilePathsArray]) => { + // Set the Project & Batch Name in params for the Promote Content Worker Action to read and process + params.project = project; + params.batchName = batchName; + + try { + const appConfig = new AppConfig(params); + const grpIds = appConfig.getConfig().grayboxUserGroups; + const vActData = await validateAction(params, grpIds, params.ignoreUserCheck); + if (vActData && vActData.code !== 200) { + logger.info(`Validation failed: ${JSON.stringify(vActData)}`); + return vActData; + } + + return ow.actions.invoke({ + name: 'graybox/promote-content-worker', + blocking: false, + result: false, + params + }).then(async (result) => { + logger.info(result); + return { + code: 200, + payload: responsePayload + }; + }).catch(async (err) => { + responsePayload = 'Failed to invoke graybox promote action'; + logger.error(`${responsePayload}: ${err}`); + return { + code: 500, + payload: responsePayload + }; + }); + } catch (err) { + responsePayload = 'Unknown error occurred'; + logger.error(`${responsePayload}: ${err}`); + responsePayload = err; + } + + return { + code: 500, + payload: responsePayload, + }; + }); + }); + } catch (err) { + responsePayload = 'Unknown error occurred'; + logger.error(`${responsePayload}: ${err}`); + responsePayload = err; + } + + return { + code: 500, + payload: responsePayload, + }; +} + +exports.main = main; diff --git a/actions/graybox/promote-worker.js b/actions/graybox/promote-worker.js index 0cfe838..86b2325 100644 --- a/actions/graybox/promote-worker.js +++ b/actions/graybox/promote-worker.js @@ -16,6 +16,7 @@ ************************************************************************* */ const fetch = require('node-fetch'); +const initFilesWrapper = require('./filesWrapper'); const { getAioLogger, handleExtension, isFilePatternMatched, toUTCStr } = require('../utils'); @@ -26,7 +27,8 @@ const Sharepoint = require('../sharepoint'); const logger = getAioLogger(); const MAX_CHILDREN = 1000; -const BATCH_REQUEST_PREVIEW = 200; +// const BATCH_REQUEST_PREVIEW = 500; +const BATCH_REQUEST_PREVIEW = 1; // TODO remove this line and uncomment the above line after testing const gbStyleExpression = 'gb-'; // graybox style expression. need to revisit if there are any more styles to be considered. const gbDomainSuffix = '-graybox'; @@ -44,8 +46,11 @@ async function main(params) { logger.info('Graybox Promote Worker invoked'); const appConfig = new AppConfig(params); - const { gbRootFolder, experienceName } = appConfig.getPayload(); - const { projectExcelPath } = appConfig.getPayload(); + const { + spToken, adminPageUri, rootFolder, gbRootFolder, promoteIgnorePaths, experienceName, projectExcelPath, draftsOnly + } = appConfig.getPayload(); + + const filesWrapper = await initFilesWrapper(logger); const sharepoint = new Sharepoint(appConfig); // Update Promote Status @@ -59,151 +64,85 @@ async function main(params) { // NOTE: This does not capture content inside the locale/expName folders yet const gbFiles = await findAllFiles(experienceName, appConfig, sharepoint); + // Create Batch Status JSON + const batchStatusJson = '{"batch_1":"initiated"}'; + const batchStatusJsonObject = JSON.parse(batchStatusJson); + + // Create Project Preview Status JSON + const previewStatusJson = []; + + // Preview Errors JSON + const projectPreviewErrorsJson = []; + // create batches to process the data - const batchArray = []; + const gbFilesBatchArray = []; + const writeBatchJsonPromises = []; for (let i = 0; i < gbFiles.length; i += BATCH_REQUEST_PREVIEW) { const arrayChunk = gbFiles.slice(i, i + BATCH_REQUEST_PREVIEW); - batchArray.push(arrayChunk); + gbFilesBatchArray.push(arrayChunk); + const batchName = `batch_${i + 1}`; + batchStatusJsonObject[`${batchName}`] = 'initiated'; + + // Each Files Batch is written to a batch_n.json file + writeBatchJsonPromises.push(filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/batches/${batchName}.json`, arrayChunk)); } + await Promise.all(writeBatchJsonPromises); + + const inputParams = {}; + inputParams.rootFolder = rootFolder; + inputParams.gbRootFolder = gbRootFolder; + inputParams.projectExcelPath = projectExcelPath; + inputParams.experienceName = experienceName; + inputParams.spToken = spToken; + inputParams.adminPageUri = adminPageUri; + inputParams.draftsOnly = draftsOnly; + inputParams.promoteIgnorePaths = promoteIgnorePaths; + + // convert the ignoreUserCheck boolean to string, so the string processing in the appConfig -> ignoreUserCheck works + inputParams.ignoreUserCheck = `${appConfig.ignoreUserCheck()}`; + + // Create Ongoing Projects JSON + const ongoingProjectsJson = `[{"project_path":"${gbRootFolder}/${experienceName}","status":"initiated"}]`; + + // Create Project Status JSON + const projectStatusJson = `{"status":"initiated", "params": ${JSON.stringify(inputParams)}}`; + const projectStatusJsonObject = JSON.parse(projectStatusJson); + + logger.info(`projectStatusJson: ${projectStatusJson}`); + + // write to JSONs to AIO Files for Ongoing Projects and Project Status + await filesWrapper.writeFile('graybox_promote/ongoing_projects.json', ongoingProjectsJson); + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/status.json`, projectStatusJsonObject); + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`, batchStatusJsonObject); + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/preview_status.json`, previewStatusJson); + await filesWrapper.writeFile(`graybox_promote${gbRootFolder}/${experienceName}/preview_errors.json`, projectPreviewErrorsJson); + + // read properties of JSON from AIO Files + const props = await filesWrapper.readProperties('graybox_promote/ongoing_projects.json'); + logger.info(`props: ${JSON.stringify(props)}`); + const projectStatusJsonProps = await filesWrapper.readProperties(`graybox_promote${gbRootFolder}/${experienceName}/status.json`); + logger.info(`status props: ${JSON.stringify(projectStatusJsonProps)}`); + const projectPreviewStatusJsonProps = await filesWrapper.readProperties(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`); + logger.info(`preivew status props: ${JSON.stringify(projectPreviewStatusJsonProps)}`); + + // read Graybox Project Json from AIO Files + const json = await filesWrapper.readFileIntoObject('graybox_promote/ongoing_projects.json'); + logger.info(`Ongoing Projects Json: ${JSON.stringify(json)}`); + const statusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/status.json`); + logger.info(`Project Status Json: ${JSON.stringify(statusJson)}`); + const projectBatchStatusJson = await filesWrapper.readFileIntoObject(`graybox_promote${gbRootFolder}/${experienceName}/batch_status.json`); + logger.info(`Project Batch Status Json: ${JSON.stringify(projectBatchStatusJson)}`); + // process data in batches - const helixUtils = new HelixUtils(appConfig); - const previewStatuses = []; - let failedPreviews = []; - const promotedPreviewStatuses = []; - let promotedFailedPreviews = []; let responsePayload; - if (helixUtils.canBulkPreview(true)) { - logger.info('Bulk Previewing Graybox files'); - const paths = []; - batchArray.forEach((batch) => { - batch.forEach((gbFile) => paths.push(handleExtension(gbFile.filePath))); - }); - previewStatuses.push(await helixUtils.bulkPreview(paths, helixUtils.getOperations().PREVIEW, experienceName, true)); - - failedPreviews = previewStatuses.flatMap((statusArray) => statusArray.filter((status) => !status.success)).map((status) => status.path); - - logger.info('Updating project excel file with status'); - const sFailedPreviews = failedPreviews.length > 0 ? `Failed Previews(Promote won't happen for these): \n${failedPreviews.join('\n')}` : ''; - const excelValues = [['Preview completed', toUTCStr(new Date()), sFailedPreviews]]; - // Update Preview Status - await sharepoint.updateExcelTable(projectExcelPath, 'PROMOTE_STATUS', excelValues); - - // Get the Helix Admin API Key for the Graybox content tree, needed for accessing (with auth) Images in graybox tree - const helixAdminApiKey = helixUtils.getAdminApiKey(true); - - // Promote Graybox files to the default content tree - const { promotes, failedPromotes } = await promoteFiles(previewStatuses, experienceName, helixAdminApiKey, sharepoint, helixUtils, appConfig); - - // Update Promote Status - const sFailedPromoteStatuses = failedPromotes.length > 0 ? `Failed Promotes: \n${failedPromotes.join('\n')}` : ''; - const promoteExcelValues = [['Promote completed', toUTCStr(new Date()), sFailedPromoteStatuses]]; - await sharepoint.updateExcelTable(projectExcelPath, 'PROMOTE_STATUS', promoteExcelValues); - - // Handle the extensions of promoted files - const promotedPaths = promotes.map((promote) => handleExtension(promote)); - - // Perform Preview of all Promoted files in the Default Content Tree - if (helixUtils.canBulkPreview(false)) { - promotedPaths.forEach((promote) => logger.info(`Promoted file in Default folder: ${promote}`)); - // Don't pass the experienceName & isGraybox params for the default content tree - promotedPreviewStatuses.push(await helixUtils.bulkPreview(promotedPaths, helixUtils.getOperations().PREVIEW)); - } - - promotedFailedPreviews = promotedPreviewStatuses.flatMap((statusArray) => statusArray.filter((status) => !status.success)).map((status) => status.path); - const sFailedPromotedPreviews = promotedFailedPreviews.length > 0 ? `Failed Promoted Previews: \n${promotedFailedPreviews.join('\n')}` : ''; - - const promotedExcelValues = [['Promoted Files Preview completed', toUTCStr(new Date()), sFailedPromotedPreviews]]; - // Update Promoted Preview Status - await sharepoint.updateExcelTable(projectExcelPath, 'PROMOTE_STATUS', promotedExcelValues); - responsePayload = 'Graybox Promote Worker action completed.'; - } else { - responsePayload = 'Bulk Preview not enabled for Graybox Content Tree'; - } + responsePayload = 'Graybox Promote Worker action completed.'; logger.info(responsePayload); return { body: responsePayload, }; } -/** -* Promote Graybox files to the default content tree - * @param {*} previewStatuses file preview statuses - * @param {*} experienceName graybox experience name - * @param {*} helixAdminApiKey helix admin api key for performing Mdast to Docx conversion - * @param {*} sharepoint sharepoint instance - * @param {*} helixUtils helix utils instance - * @param {*} appConfig app config instance - * @returns JSON array of successful & failed promotes - */ -async function promoteFiles(previewStatuses, experienceName, helixAdminApiKey, sharepoint, helixUtils, appConfig) { - const promotes = []; - const failedPromotes = []; - const options = {}; - // Passing isGraybox param true to fetch graybox Hlx Admin API Key - const grayboxHlxAdminApiKey = helixUtils.getAdminApiKey(true); - if (grayboxHlxAdminApiKey) { - options.headers = new fetch.Headers(); - options.headers.append('Authorization', `token ${grayboxHlxAdminApiKey}`); - } - - // iterate through preview statuses, generate docx files and promote them - const allPromises = previewStatuses.map(async (status) => { - // check if status is an array and iterate through the array - if (Array.isArray(status)) { - const promises = status.map(async (stat) => { - if (stat.success && stat.mdPath) { - const response = await sharepoint.fetchWithRetry(`${stat.mdPath}`, options); - const content = await response.text(); - let docx; - const sp = await appConfig.getSpConfig(); - - if (content.includes(experienceName) || content.includes(gbStyleExpression) || content.includes(gbDomainSuffix)) { - // Process the Graybox Styles and Links with Mdast to Docx conversion - docx = await updateDocument(content, experienceName, helixAdminApiKey); - if (docx) { - logger.info(`Docx file generated for ${stat.path}`); - // Save file Destination full path with file name and extension - const destinationFilePath = `${stat.path.substring(0, stat.path.lastIndexOf('/') + 1).replace('/'.concat(experienceName), '')}${stat.fileName}`; - const saveStatus = await sharepoint.saveFileSimple(docx, destinationFilePath); - - if (saveStatus?.success) { - promotes.push(destinationFilePath); - } else if (saveStatus?.errorMsg?.includes('File is locked')) { - failedPromotes.push(`${destinationFilePath} (locked file)`); - } else { - failedPromotes.push(destinationFilePath); - } - } else { - logger.error(`Error generating docx file for ${stat.path}`); - } - } else { - const copySourceFilePath = `${stat.path.substring(0, stat.path.lastIndexOf('/') + 1)}${stat.fileName}`; // Copy Source full path with file name and extension - const copyDestinationFolder = `${stat.path.substring(0, stat.path.lastIndexOf('/')).replace('/'.concat(experienceName), '')}`; // Copy Destination folder path, no file name - const destFilePath = `${copyDestinationFolder}/${stat.fileName}`; - - // Download the grayboxed file and save it to default content location - const { fileDownloadUrl } = await sharepoint.getFileData(copySourceFilePath, true); - const file = await sharepoint.getFileUsingDownloadUrl(fileDownloadUrl); - const saveStatus = await sharepoint.saveFileSimple(file, destFilePath); - - if (saveStatus?.success) { - promotes.push(destFilePath); - } else if (saveStatus?.errorMsg?.includes('File is locked')) { - failedPromotes.push(`${destFilePath} (locked file)`); - } else { - failedPromotes.push(destFilePath); - } - } - } - }); - await Promise.all(promises); // await all async functions in the array are executed, before updating the status in the graybox project excel - } - }); - await Promise.all(allPromises); // await all async functions in the array are executed, before updating the status in the graybox project excel - return { promotes, failedPromotes }; -} - /** * Find all files in the Graybox tree to promote. */ diff --git a/actions/graybox/validateAction.js b/actions/graybox/validateAction.js index 5e2ab6e..a7abdd1 100644 --- a/actions/graybox/validateAction.js +++ b/actions/graybox/validateAction.js @@ -17,6 +17,7 @@ const AppConfig = require('../appConfig'); const GrayboxUser = require('../grayboxUser'); +const { getAioLogger } = require('../utils'); function isGrayboxParamsValid(params) { const { @@ -41,6 +42,7 @@ async function isUserAuthorized(params, grpIds) { const appConfig = new AppConfig(params); const grayboxUser = new GrayboxUser({ appConfig }); const found = await grayboxUser.isInGroups(grpIds); + getAioLogger().info(`User is authorized: ${found}`); return found; } diff --git a/app.config.yaml b/app.config.yaml index f1cb549..0ae9c48 100644 --- a/app.config.yaml +++ b/app.config.yaml @@ -13,6 +13,7 @@ application: certPassword: $CERT_PASSWORD certKey: $CERT_KEY certThumbprint: $CERT_THUMB_PRINT + driveId: $DRIVE_ID enablePreview: $ENABLE_PREVIEW groupCheckUrl: $GROUP_CHECK_URL grayboxUserGroups: $GRAYBOX_USER_GROUPS @@ -25,7 +26,7 @@ application: web: 'yes' runtime: nodejs:18 inputs: - LOG_LEVEL: debug + LOG_LEVEL: debug promote-worker: function: actions/graybox/promote-worker.js web: 'no' @@ -35,3 +36,105 @@ application: limits: timeout: 3600000 memorySize: 2048 + preview-worker: + function: actions/graybox/preview-worker.js + web: 'no' + runtime: nodejs:18 + inputs: + LOG_LEVEL: debug + limits: + timeout: 3600000 + memorySize: 2048 + process-docx-worker: + function: actions/graybox/process-docx-worker.js + web: 'no' + runtime: nodejs:18 + inputs: + LOG_LEVEL: debug + limits: + timeout: 3600000 + memorySize: 2048 + promote-content-worker: + function: actions/graybox/promote-content-worker.js + web: 'no' + runtime: nodejs:18 + inputs: + LOG_LEVEL: debug + limits: + timeout: 3600000 + memorySize: 2048 + copy-content-worker: + function: actions/graybox/copy-content-worker.js + web: 'no' + runtime: nodejs:18 + inputs: + LOG_LEVEL: debug + limits: + timeout: 3600000 + memorySize: 2048 + preview-sched: + function: actions/graybox/preview-sched.js + web: 'no' + runtime: 'nodejs:18' + inputs: + LOG_LEVEL: debug + limits: + timeout: 900000 + memorySize: 2048 + annotations: + require-adobe-auth: false + final: true + process-docx-sched: + function: actions/graybox/process-docx-sched.js + web: 'no' + runtime: 'nodejs:18' + inputs: + LOG_LEVEL: debug + limits: + timeout: 900000 + memorySize: 2048 + annotations: + require-adobe-auth: false + final: true + promote-sched: + function: actions/graybox/promote-sched.js + web: 'no' + runtime: 'nodejs:18' + inputs: + LOG_LEVEL: debug + limits: + timeout: 900000 + memorySize: 2048 + annotations: + require-adobe-auth: false + final: true + copy-sched: + function: actions/graybox/copy-sched.js + web: 'no' + runtime: 'nodejs:18' + inputs: + LOG_LEVEL: debug + limits: + timeout: 900000 + memorySize: 2048 + annotations: + require-adobe-auth: false + final: true + triggers: + everyMin: + feed: /whisk.system/alarms/interval + inputs: + minutes: 1 + rules: + everyMinRule: + trigger: everyMin + action: preview-sched + everyMinProcessDocxRule: + trigger: everyMin + action: process-docx-sched + everyMinPromoteDocxRule: + trigger: everyMin + action: promote-sched + everyMinCopyDocxRule: + trigger: everyMin + action: copy-sched diff --git a/package-lock.json b/package-lock.json index 55de4d3..9867da9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.1", "dependencies": { "@adobe/aio-lib-core-logging": "^3.0.1", + "@adobe/aio-lib-files": "^4.0.1", "@adobe/aio-sdk": "^5", "@azure/msal-node": "^2.6.5", "milo-md2docx": "^1.8.0", @@ -660,9 +661,9 @@ } }, "node_modules/@adobe/aio-lib-core-networking": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/@adobe/aio-lib-core-networking/-/aio-lib-core-networking-5.0.2.tgz", - "integrity": "sha512-Jj5/feIQQm2O59Aprp+tXUYQeJ5zKJrZGQhhD4Du9ApZ5Ulz1Qq360thQg9nWMdy8yAWWbA/bY2WmOm9/uqudg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@adobe/aio-lib-core-networking/-/aio-lib-core-networking-5.0.1.tgz", + "integrity": "sha512-F/RHorqDXTGuSgOrxsAI6dNEvTWWT86vArfWQPh/UJ3YuERgkgdHt9FYFcHmLARBLDqGVkRaFYbjMdL6YHetew==", "dependencies": { "@adobe/aio-lib-core-config": "^5.0.0", "@adobe/aio-lib-core-errors": "^4.0.0", @@ -677,11 +678,70 @@ "node": ">=18" } }, + "node_modules/@adobe/aio-lib-core-networking/node_modules/@adobe/aio-lib-core-config": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@adobe/aio-lib-core-config/-/aio-lib-core-config-5.0.1.tgz", + "integrity": "sha512-OQmQublmy/uXM1HC6qXfxSAXEl85nExh/yiajlEfJheKuJ9iPWwVWXR5vBHVVDlOXgWEVMWRUQPMIUu1lmR5lA==", + "dependencies": { + "debug": "^4.1.1", + "deepmerge": "^4.0.0", + "dotenv": "16.3.1", + "hjson": "^3.1.2", + "js-yaml": "^4.1.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@adobe/aio-lib-core-networking/node_modules/@adobe/aio-lib-core-errors": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@adobe/aio-lib-core-errors/-/aio-lib-core-errors-4.0.1.tgz", + "integrity": "sha512-zrQm9TJh13wEHH5O2TQAUQvYGGe01R9DHzKy+b6B0URbl2lcuqXyNiUx896lpcgXD2bzUoH7ARRH97aCW2tlfw==" + }, + "node_modules/@adobe/aio-lib-core-networking/node_modules/@adobe/aio-lib-core-logging": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@adobe/aio-lib-core-logging/-/aio-lib-core-logging-3.0.1.tgz", + "integrity": "sha512-WvhFXy5sCIBHwGNP6QS2LiGVWeFb6vxKiZ62W0ahwN5SqHqaBoimBDvTysdH9gANGLShELPPT2gb4255sElf5w==", + "dependencies": { + "debug": "^4.1.1", + "winston": "^3.2.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@adobe/aio-lib-core-networking/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "node_modules/@adobe/aio-lib-core-networking/node_modules/dotenv": { + "version": "16.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", + "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/motdotla/dotenv?sponsor=1" + } + }, "node_modules/@adobe/aio-lib-core-networking/node_modules/fetch-retry": { "version": "5.0.6", "resolved": "https://registry.npmjs.org/fetch-retry/-/fetch-retry-5.0.6.tgz", "integrity": "sha512-3yurQZ2hD9VISAhJJP9bpYFNQrHHBXE2JxxjY5aLEcDi46RmAzJE2OC9FAde0yis5ElW0jTTzs0zfg/Cca4XqQ==" }, + "node_modules/@adobe/aio-lib-core-networking/node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/@adobe/aio-lib-core-tvm": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@adobe/aio-lib-core-tvm/-/aio-lib-core-tvm-4.0.2.tgz", @@ -700,6 +760,23 @@ "node": ">=18" } }, + "node_modules/@adobe/aio-lib-core-tvm/node_modules/@adobe/aio-lib-core-errors": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@adobe/aio-lib-core-errors/-/aio-lib-core-errors-4.0.1.tgz", + "integrity": "sha512-zrQm9TJh13wEHH5O2TQAUQvYGGe01R9DHzKy+b6B0URbl2lcuqXyNiUx896lpcgXD2bzUoH7ARRH97aCW2tlfw==" + }, + "node_modules/@adobe/aio-lib-core-tvm/node_modules/@adobe/aio-lib-core-logging": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@adobe/aio-lib-core-logging/-/aio-lib-core-logging-3.0.1.tgz", + "integrity": "sha512-WvhFXy5sCIBHwGNP6QS2LiGVWeFb6vxKiZ62W0ahwN5SqHqaBoimBDvTysdH9gANGLShELPPT2gb4255sElf5w==", + "dependencies": { + "debug": "^4.1.1", + "winston": "^3.2.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@adobe/aio-lib-core-tvm/node_modules/upath": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/upath/-/upath-2.0.1.tgz", @@ -748,6 +825,60 @@ "node": ">=18" } }, + "node_modules/@adobe/aio-lib-env/node_modules/@adobe/aio-lib-core-config": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@adobe/aio-lib-core-config/-/aio-lib-core-config-5.0.1.tgz", + "integrity": "sha512-OQmQublmy/uXM1HC6qXfxSAXEl85nExh/yiajlEfJheKuJ9iPWwVWXR5vBHVVDlOXgWEVMWRUQPMIUu1lmR5lA==", + "dependencies": { + "debug": "^4.1.1", + "deepmerge": "^4.0.0", + "dotenv": "16.3.1", + "hjson": "^3.1.2", + "js-yaml": "^4.1.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@adobe/aio-lib-env/node_modules/@adobe/aio-lib-core-logging": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@adobe/aio-lib-core-logging/-/aio-lib-core-logging-3.0.1.tgz", + "integrity": "sha512-WvhFXy5sCIBHwGNP6QS2LiGVWeFb6vxKiZ62W0ahwN5SqHqaBoimBDvTysdH9gANGLShELPPT2gb4255sElf5w==", + "dependencies": { + "debug": "^4.1.1", + "winston": "^3.2.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@adobe/aio-lib-env/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "node_modules/@adobe/aio-lib-env/node_modules/dotenv": { + "version": "16.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", + "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/motdotla/dotenv?sponsor=1" + } + }, + "node_modules/@adobe/aio-lib-env/node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/@adobe/aio-lib-events": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@adobe/aio-lib-events/-/aio-lib-events-4.0.1.tgz", @@ -792,6 +923,11 @@ "node": ">=18" } }, + "node_modules/@adobe/aio-lib-files/node_modules/@adobe/aio-lib-core-errors": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@adobe/aio-lib-core-errors/-/aio-lib-core-errors-4.0.1.tgz", + "integrity": "sha512-zrQm9TJh13wEHH5O2TQAUQvYGGe01R9DHzKy+b6B0URbl2lcuqXyNiUx896lpcgXD2bzUoH7ARRH97aCW2tlfw==" + }, "node_modules/@adobe/aio-lib-files/node_modules/uuid": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", @@ -886,6 +1022,25 @@ "yauzl": "2.10.0" } }, + "node_modules/@adobe/helix-docx2md/node_modules/@adobe/mammoth": { + "version": "1.5.1-bleeding.2", + "resolved": "https://registry.npmjs.org/@adobe/mammoth/-/mammoth-1.5.1-bleeding.2.tgz", + "integrity": "sha512-quhwkeOckKfPv3ubpi+OZImtJeJ9gyHWD//QfDWaY4USsL059Y1uB/Kbzw7RGAWYaNGQiq2vI2jCx6DH3LKeiQ==", + "dependencies": { + "@xmldom/xmldom": "^0.8.6", + "argparse": "~1.0.3", + "bluebird": "~3.4.0", + "dingbat-to-unicode": "^1.0.1", + "jszip": "^3.7.1", + "lop": "^0.4.1", + "path-is-absolute": "^1.0.0", + "underscore": "^1.13.1", + "xmlbuilder": "^10.0.0" + }, + "bin": { + "mammoth": "bin/mammoth" + } + }, "node_modules/@adobe/helix-markdown-support": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/@adobe/helix-markdown-support/-/helix-markdown-support-7.0.0.tgz", @@ -992,17 +1147,17 @@ } }, "node_modules/@adobe/mammoth": { - "version": "1.5.1-bleeding.2", - "resolved": "https://registry.npmjs.org/@adobe/mammoth/-/mammoth-1.5.1-bleeding.2.tgz", - "integrity": "sha512-quhwkeOckKfPv3ubpi+OZImtJeJ9gyHWD//QfDWaY4USsL059Y1uB/Kbzw7RGAWYaNGQiq2vI2jCx6DH3LKeiQ==", + "version": "1.5.1-bleeding.1", + "resolved": "https://registry.npmjs.org/@adobe/mammoth/-/mammoth-1.5.1-bleeding.1.tgz", + "integrity": "sha512-wI9kMxh1ZvgVg/eQuFCuPUYevNRFPShNh5Iq2oDglfI4dJ44NZ7TzTzAfLMADdqd3YgRzAj37QRi+2aOSXwcUw==", "dependencies": { - "@xmldom/xmldom": "^0.8.6", "argparse": "~1.0.3", "bluebird": "~3.4.0", "dingbat-to-unicode": "^1.0.1", "jszip": "^3.7.1", "lop": "^0.4.1", "path-is-absolute": "^1.0.0", + "sax": "~1.1.1", "underscore": "^1.13.1", "xmlbuilder": "^10.0.0" }, @@ -1010,13 +1165,10 @@ "mammoth": "bin/mammoth" } }, - "node_modules/@adobe/mammoth/node_modules/xmlbuilder": { - "version": "10.1.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-10.1.1.tgz", - "integrity": "sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==", - "engines": { - "node": ">=4.0" - } + "node_modules/@adobe/mammoth/node_modules/sax": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.1.6.tgz", + "integrity": "sha512-8zci48uUQyfqynGDSkUMD7FCJB96hwLnlZOXlgs1l3TX+LW27t3psSWKUxC0fxVgA86i8tL4NwGcY1h/6t3ESg==" }, "node_modules/@adobe/mdast-util-gridtables": { "version": "3.0.1", @@ -14235,6 +14387,14 @@ "xml-js": "bin/cli.js" } }, + "node_modules/xmlbuilder": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-10.1.1.tgz", + "integrity": "sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 840decb..bf8778e 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "private": true, "dependencies": { "@adobe/aio-lib-core-logging": "^3.0.1", + "@adobe/aio-lib-files": "^4.0.1", "@adobe/aio-sdk": "^5", "@azure/msal-node": "^2.6.5", "milo-md2docx": "^1.8.0",