Skip to content

Commit

Permalink
MWPW-145191 - Promote Graybox files after MDAST cleanup (#11)
Browse files Browse the repository at this point in the history 
- Uploading files after MDAST processing to the Default folder from Graybox folder
- Promote Copy files without graybox styles from Graybox to Default Folder
  • Loading branch information
@arshadparwaiz
arshadparwaiz authored May 14, 2024
1 parent e9920b9 commit 8c34e64
Show file tree
Hide file tree
Showing 9 changed files with 2,110 additions and 89 deletions.
2 changes: 1 addition & 1 deletion actions/config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
* is strictly forbidden unless prior written permission is obtained
* from Adobe.
************************************************************************* */

const appConfig = require('./appConfig');

const GRAPH_API = 'https://graph.microsoft.com/v1.0';
Expand Down Expand Up @@ -103,6 +102,7 @@ function getHelixAdminConfig() {
async function getConfig() {
if (appConfig.getUrlInfo().isValid()) {
const applicationConfig = appConfig.getConfig();

return {
sp: getSharepointConfig(applicationConfig),
admin: getHelixAdminConfig(),
Expand Down
106 changes: 59 additions & 47 deletions actions/docxUpdater.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,40 +17,44 @@
const parseMarkdown = require('milo-parse-markdown').default;
const { mdast2docx } = require('../node_modules/milo-md2docx/lib/index');
const { getAioLogger } = require('./utils');
const { fetchWithRetry } = require('./sharepoint');

const gbStyleExpression = 'gb-';//graybox style expression. need to revisit if there are any more styles to be considered.
const DEFAULT_STYLES = require('../defaultstyles.xml');

const gbStyleExpression = 'gb-'; // graybox style expression. need to revisit if there are any more styles to be considered.
const emptyString = '';
const grayboxStylesRegex = new RegExp('gb-[a-zA-Z0-9,._-]*', 'g');
const gbDomainSuffix = '-graybox';
const logger = getAioLogger();
let firstGtRows = [];


/**
* Updates a document based on the provided Markdown file path, experience name, and options.
* @param {string} mdPath - The path to the Markdown file.
* @param {string} experienceName - The name of the experience.
* @param {object} options - The options for fetching the Markdown file.
* @returns {Promise} - A promise that resolves to the generated Docx file.
*/
async function updateDocument(mdPath, expName, options = {}){
async function updateDocument(content, expName, hlxAdminApiKey) {
firstGtRows = [];
const response = await fetchWithRetry(`${mdPath}`, options);
const content = await response.text();
if (content.includes(expName) || content.includes(gbStyleExpression) || content.includes(gbDomainSuffix)) {
const state = { content: { data: content }, log: '' };
await parseMarkdown(state);
const { mdast } = state.content;
updateExperienceNameFromLinks(mdast.children, expName);
logger.info('Experience name removed from links');
iterateGtRowsToReplaceStyles();
logger.info('Graybox styles removed');
//generated docx file from updated mdast
const docx = await generateDocxFromMdast(mdast);
//TODO promote this docx file
logger.info('Mdast to Docx file conversion done');
let docx;

const state = { content: { data: content }, log: '' };
await parseMarkdown(state);
const { mdast } = state.content;
updateExperienceNameFromLinks(mdast.children, expName);

iterateGtRowsToReplaceStyles();

try {
// generated docx file from updated mdast
docx = await generateDocxFromMdast(mdast, hlxAdminApiKey);
} catch (err) {
// Mostly bad string ignored
logger.debug(`Error while generating docxfromdast ${err}`);
}

logger.info('Mdast to Docx file conversion done');
return docx;
}

/**
Expand All @@ -62,55 +66,55 @@ async function updateDocument(mdPath, expName, options = {}){
const updateExperienceNameFromLinks = (mdast, expName) => {
if (mdast) {
mdast.forEach((child) => {
if (child.type === 'gridTable') {
firstGtRows.push(findFirstGtRowInNode(child));
}
//remove experience name from links on the document
if (child.type === 'link' && child.url && (child.url.includes(expName) || child.url.includes(gbDomainSuffix))) {
child.url = child.url.replaceAll(`/${expName}/`, '/').replaceAll(gbDomainSuffix, emptyString);
logger.info(`Link updated: ${child.url}`);
}
if (child.children) {
updateExperienceNameFromLinks(child.children, expName);
}
if (child.type === 'gridTable') {
firstGtRows.push(findFirstGtRowInNode(child));
}
// remove experience name from links on the document
if (child.type === 'link' && child.url && (child.url.includes(expName) || child.url.includes(gbDomainSuffix))) {
child.url = child.url.replaceAll(`/${expName}/`, '/').replaceAll(gbDomainSuffix, emptyString);
}
if (child.children) {
updateExperienceNameFromLinks(child.children, expName);
}
);
});
}
}
};

/**
* Helper function, iterates through the firstGtRows array and replaces graybox styles for each row.
*/
const iterateGtRowsToReplaceStyles = () => {
firstGtRows.forEach((gtRow) => {
if (gtRow && gtRow.children) {
replaceGrayboxStyles(gtRow);
}
});
}
try {
firstGtRows.forEach((gtRow) => {
if (gtRow && gtRow.children) {
replaceGrayboxStyles(gtRow);
}
});
} catch (err) {
// Mostly bad string ignored
logger().debug(`Error while iterating GTRows to replaces styles ${err}`);
}
};

/**
* Replaces all graybox styles from blocks and text.
*
*
* @param {object} node - The node to process.
* @returns {void}
*/
const replaceGrayboxStyles = (node) => {
//replace all graybox styles from blocks and text
// replace all graybox styles from blocks and text
if (node && node.type === 'text' && node.value && node.value.includes(gbStyleExpression)) {
logger.info(node);
node.value = node.value.replace(grayboxStylesRegex, emptyString)
.replace('()', emptyString).replace(', )', ')');
logger.info('updated value>> ');
logger.info(node);
return;
}
if (node.children) {
node.children.forEach((child) => {
replaceGrayboxStyles(child);
});
}
}
};

/**
* Finds the first 'gtRow' node in the given node or its children.
Expand All @@ -126,17 +130,25 @@ function findFirstGtRowInNode(node) {
return findFirstGtRowInNode(child);
}
}
return null;
}


/**
* Generate a Docx file from the given mdast.
* @param {Object} mdast - The mdast representing the document.
* @returns {Promise} A promise that resolves to the generated Docx file.
*/
async function generateDocxFromMdast(mdast) {
logger.info('Docx file Docx file generation from mdast started...');
return await mdast2docx(mdast);
async function generateDocxFromMdast(mdast, hlxAdminApiKey) {
const options = {
stylesXML: DEFAULT_STYLES,
auth: {
authorization: `token ${hlxAdminApiKey}`,
}
};

const docx = await mdast2docx(mdast, options);

return docx;
}

module.exports = updateDocument;
82 changes: 62 additions & 20 deletions actions/graybox/promote-worker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,21 +15,24 @@
* from Adobe.
************************************************************************* */

const { getAioLogger, handleExtension, logMemUsage, delay, isFilePatternMatched, toUTCStr } = require('../utils');
const fetch = require('node-fetch');
const {
getAioLogger, handleExtension, isFilePatternMatched, toUTCStr
} = require('../utils');
const appConfig = require('../appConfig');
const { getConfig } = require('../config');
const { getAuthorizedRequestOption, fetchWithRetry, updateExcelTable, bulkCreateFolders } = require('../sharepoint');
const { getAuthorizedRequestOption, fetchWithRetry, updateExcelTable } = require('../sharepoint');
const helixUtils = require('../helixUtils');
const sharepointAuth = require('../sharepointAuth');
const updateDocument = require('../docxUpdater');
const fetch = require('node-fetch');

const { saveFile, copyFile, promoteCopy } = require('../sharepoint');

const logger = getAioLogger();
const MAX_CHILDREN = 1000;
const IS_GRAYBOX = true;
const BATCH_REQUEST_PREVIEW = 200;
const DELAY_TIME_COPY = 3000;

const gbStyleExpression = 'gb-'; // graybox style expression. need to revisit if there are any more styles to be considered.
const gbDomainSuffix = '-graybox';

async function main(params) {
logger.info('Graybox Promote Worker invoked');
Expand All @@ -54,41 +57,71 @@ async function main(params) {
logger.info(`Files in graybox folder in ${experienceName}`);
logger.info(JSON.stringify(gbFiles));


// create batches to process the data
const batchArray = [];
for (let i = 0; i < gbFiles.length; i += BATCH_REQUEST_PREVIEW) {
const arrayChunk = gbFiles.slice(i, i + BATCH_REQUEST_PREVIEW);
batchArray.push(arrayChunk);
}

// process data in batches
const previewStatuses = [];
let failedPreviews = '';
let failedPreviews = [];
const promoteStatuses = [];
const failedPromoteStatuses = [];
if (helixUtils.canBulkPreview()) {
const paths = [];
batchArray.forEach((batch) => {
batch.forEach((gbFile) => paths.push(handleExtension(gbFile.filePath)));
});
previewStatuses.push(await helixUtils.bulkPreview(paths, helixUtils.getOperations().PREVIEW, experienceName));
logger.info(`Preview Statuses >> ${JSON.stringify(previewStatuses)}`);
const failedPreviews = previewStatuses.filter((status) => !status.success).map((status) => status.path);
const urlInfo = appConfig.getUrlInfo();
failedPreviews = previewStatuses.filter((status) => !status.success).map((status) => status.path);

const helixAdminApiKey = helixUtils.getAdminApiKey();

const options = {};
if (helixUtils.getAdminApiKey()) {
options.headers = new fetch.Headers();
options.headers.append('Authorization', `token ${helixUtils.getAdminApiKey()}`);
}

// iterate through preview statuses and log success
previewStatuses.forEach((status) => {
//check if status is an array and iterate through the array
// check if status is an array and iterate through the array
if (Array.isArray(status)) {
status.forEach((stat) => {
logger.info(`status >> ${JSON.stringify(stat)}`);
status.forEach(async (stat) => {
if (stat.success && stat.mdPath) {
logger.info(`Preview success and mdPath for file: ${stat.path} & ${stat.mdPath}`);
updateDocument(stat.mdPath, experienceName, options);
const response = await fetchWithRetry(`${stat.mdPath}`, options);
const content = await response.text();
let docx;
const { sp } = await getConfig();

if (content.includes(experienceName) || content.includes(gbStyleExpression) || content.includes(gbDomainSuffix)) {
docx = await updateDocument(content, experienceName, helixAdminApiKey);
if (docx) {
// Save file Destination full path with file name and extension
const destinationFilePath = `${stat.path.substring(0, stat.path.lastIndexOf('/') + 1).replace('/'.concat(experienceName), '')}${stat.fileName}`;

const saveStatus = await saveFile(docx, destinationFilePath, sp);
if (saveStatus && saveStatus.success === true) {
promoteStatuses.push(destinationFilePath);
} else {
failedPromoteStatuses.push(destinationFilePath);
}
} else {
logger.error(`Error generating docx file for ${stat.path}`);
}
} else {
const copySourceFilePath = `${stat.path.substring(0, stat.path.lastIndexOf('/') + 1)}${stat.fileName}`; // Copy Source full path with file name and extension
const copyDestinationFolder = `${stat.path.substring(0, stat.path.lastIndexOf('/')).replace('/'.concat(experienceName), '')}`; // Copy Destination folder path, no file name
const promoteCopyFileStatus = await promoteCopy(copySourceFilePath, copyDestinationFolder, stat.fileName, sp);

if (promoteCopyFileStatus) {
promoteStatuses.push(`${copyDestinationFolder}/${stat.fileName}`);
} else {
failedPromoteStatuses.push(`${copyDestinationFolder}/${stat.fileName}`);
}
}
}
});
}
Expand All @@ -99,10 +132,19 @@ async function main(params) {
logger.info('Updating project excel file with status');
const curreDateTime = new Date();
const { projectExcelPath } = appConfig.getPayload();
const sFailedPreviews = failedPreviews.length > 0 ? 'Failed Previews: \n' + failedPreviews.join('\n') : '';
const sFailedPreviews = failedPreviews.length > 0 ? `Failed Previews: \n${failedPreviews.join('\n')}` : '';
const excelValues = [['Preview', toUTCStr(curreDateTime), sFailedPreviews]];
// Update Preview Status
await updateExcelTable(projectExcelPath, 'PROMOTE_STATUS', excelValues, IS_GRAYBOX);
logger.info('Project excel file updated with promote status.');

// Update Promote Status
const sPromoteStatuses = promoteStatuses.length > 0 ? `Promotes: \n${promoteStatuses.join('\n')}` : '';
const sFailedPromoteStatuses = failedPromoteStatuses.length > 0 ? `Failed Promotes: \n${failedPromoteStatuses.join('\n')}` : '';
const promoteExcelValues = [['Promote', toUTCStr(curreDateTime), sPromoteStatuses]];
const failedPromoteExcelValues = [['Promote', toUTCStr(curreDateTime), sFailedPromoteStatuses]];

await updateExcelTable(projectExcelPath, 'PROMOTE_STATUS', promoteExcelValues, IS_GRAYBOX);
await updateExcelTable(projectExcelPath, 'PROMOTE_STATUS', failedPromoteExcelValues, IS_GRAYBOX);

const responsePayload = 'Graybox Promote Worker action completed.';
return exitAction({
Expand Down
17 changes: 10 additions & 7 deletions actions/helixUtils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,11 @@ class HelixUtils {
* @returns List of path with preview/pubish status e.g. [{path:'/draft/file1', success: true}..]
*/
async bulkPreview(paths, operation, grayboxExperienceName, retryAttempt = 1) {
let prevStatuses = paths.filter((p) => p).map((path) => ({ success: false, path, resourcePath: '', responseCode: '' }));
let prevStatuses = paths.filter((p) => p).map((path) => (
{
success: false, path, fileName: '', resourcePath: '', responseCode: ''
}
));
if (!prevStatuses.length) {
return prevStatuses;
}
Expand Down Expand Up @@ -98,14 +102,14 @@ class HelixUtils {
const jobResp = await response.json();
const jobName = jobResp.job?.name;
if (jobName) {
logger.info(`check again jobName : ${jobName} operation : ${operation} repo : ${repo}`);
const jobStatus = await this.bulkJobStatus(jobName, operation, repo);
logger.info(`jobStatus : ${JSON.stringify(jobStatus)}`);
prevStatuses.forEach((e) => {
logger.info(`Job details : ${jobName} / ${jobResp.messageId} / ${jobResp.job?.state}`);
if (jobStatus[e.path]?.success) {
e.success = true;
e.fileName = jobStatus[e.path]?.fileName;
e.resourcePath = jobStatus[e.path]?.resourcePath;

e.mdPath = `https://${urlInfo.getBranch()}--${urlInfo.getRepo()}--${urlInfo.getOwner()}.hlx.page${e.resourcePath}`;
}
e.responseCode = jobStatus[e.path]?.responseCode;
Expand All @@ -131,7 +135,6 @@ class HelixUtils {
* @returns List of path with preview/pubish status e.g. ['/draft/file1': {success: true}..]
*/
async bulkJobStatus(jobName, operation, repo, bulkPreviewStatus = {}, retryAttempt = 1) {
logger.info(`Checking job status of ${jobName} for ${operation}`);
try {
const { helixAdminApiKeys } = appConfig.getConfig();
const options = {};
Expand All @@ -148,10 +151,10 @@ class HelixUtils {
await this.bulkJobStatus(jobName, operation, repo, bulkPreviewStatus, retryAttempt + 1);
} else if (response.ok) {
const jobStatusJson = await response.json();
logger.info(`jobStatusJson ${JSON.stringify(jobStatusJson)}`);
logger.info(`${operation} progress ${JSON.stringify(jobStatusJson.progress)}`);
jobStatusJson.data?.resources?.forEach((rs) => {
bulkPreviewStatus[rs.path] = { success: JOB_STATUS_CODES.includes(rs.status), resourcePath: rs?.resourcePath, responseCode: rs.status };
bulkPreviewStatus[rs.path] = {
success: JOB_STATUS_CODES.includes(rs.status), fileName: rs?.source?.name, resourcePath: rs?.resourcePath, responseCode: rs.status
};
});
if (jobStatusJson.state !== 'stopped' && !jobStatusJson.cancelled &&
retryAttempt <= appConfig.getConfig().maxBulkPreviewChecks) {
Expand Down
Loading

0 comments on commit 8c34e64

Please sign in to comment.