From 8852fa36a3fed4d40be2e9bce3d09eceb42c3c9b Mon Sep 17 00:00:00 2001 From: Phil Guinchard Date: Wed, 6 Nov 2024 11:00:49 -0500 Subject: [PATCH] Adding new flags for scrubbing --- README.md | 53 +++++++++++------- libs/general.js | 36 ++++++++++-- libs/libnbdbackup.js | 129 +++++++++++++++++++------------------------ libs/qemu-img.js | 17 +++--- libs/virsh.js | 11 +++- package-lock.json | 4 +- package.json | 4 +- vmsnap.js | 5 +- 8 files changed, 150 insertions(+), 109 deletions(-) diff --git a/README.md b/README.md index 9ca729b..e4d4109 100644 --- a/README.md +++ b/README.md @@ -75,21 +75,23 @@ command. Doing so will install VMSnap which includes a vmsnap bin. The following CLI switches are available when invoking VMSnap. -| Switch | Status | Backup | Scrub | Type | Examples/Notes | -|------------|--------|--------|--------|---------|-------------------------------------------------------------------| -| domains | ✅ | ✅ | ✅ | string | "vm1" or "vm1,vm2,etc" or "*" | -| status | ✅ | - | - | boolean | Querys the domain(s) | -| backup | - | ✅ | - | boolean | Does an incremental backup (if possible) | -| scrub | - | - | ✅ | boolean | Cleans checkpoints and bitmaps off of the domain | -| output | ✅ | ✅ | - | string | A full path to a directory where backups are placed | -| verbose | ✅ | - | - | boolean | Prints out extra information when running a status check | -| machine | ✅ | - | - | boolean | Removes some output from the status command | -| json | ✅ | - | - | boolean | Outputs the status command is JSON | -| yaml | ✅ | - | - | boolean | Output YAML from the status command (aliased to `--yml`) | -| raw | - | ✅ | - | boolean | Enables raw disk handling | -| groupBy | ✅ | ✅ | - | string | Defines how backups are grouped on disk (month, quarter, or year) | -| prune | - | ✅ | - | boolean | Rotates backups by **deleting** last periods backup* | -| pretty | ✅ | - | - | boolean | Pretty prints disk sizes (42.6 GB, 120 GB, etc) | +| Switch | Status | Backup | Scrub | Type | Examples/Notes | +|----------------|--------|--------|--------|---------|--------------------------------------------------------------------| +| domains | ✅ | ✅ | ✅ | string | "vm1" or "vm1,vm2,etc" or "*" | +| status | ✅ | - | - | boolean | Querys the domain(s) | +| backup | - | ✅ | - | boolean | Does an incremental backup (if possible) | +| scrub | - | - | ✅ | boolean | Cleans checkpoints and bitmaps off of the domain | +| output | ✅ | ✅ | - | string | A full path to a directory where backups are placed | +| verbose | ✅ | - | - | boolean | Prints out extra information when running a status check | +| machine | ✅ | - | - | boolean | Removes some output from the status command | +| json | ✅ | - | - | boolean | Outputs the status command is JSON | +| yaml | ✅ | - | - | boolean | Output YAML from the status command (aliased to `--yml`) | +| raw | - | ✅ | - | boolean | Enables raw disk handling | +| groupBy | ✅ | ✅ | - | string | Defines how backups are grouped on disk (month, quarter, or year) | +| prune | - | ✅ | - | boolean | Rotates backups by **deleting** last periods backup* | +| pretty | ✅ | - | - | boolean | Pretty prints disk sizes (42.6 GB, 120 GB, etc) | +| checkpointName | - | - | ✅ | boolean | The name of the checkpoint to delete (no effect when scrubType=*) | +| scrubType | - | - | ✅ | boolean | The type of item to scrub (checkpoint, bitmap, both, or * for ALL) | *\*This happens on or after the the middle of the current period (15 days monthly, 45 days quarterly or 180 yearly)* @@ -180,8 +182,8 @@ on disk. Look at the table below for more information. #### Pruning (Caution) -**Note:** Pruning is destructive. Be careful when using it and check your -backups frequently! +> **Note:** Pruning is destructive. Be careful when using it and check your + backups frequently! Pruning backups may be done by setting `--prune` on the backup command. This flag will automatically delete last periods backup once the middle of the @@ -198,12 +200,23 @@ You can turn on raw disk handling by setting the `--raw` flag. ### Scrubbing -**Note:** This is an inherently destructive action, be careful! +> **Note:** These commands are inherently destructive, be careful! -To scrub a VM of checkpoints and bitmaps: +It is occasionally useful to be able to scrub one or more checkpoints or bitmaps +from your domain. Doing so is fairly straight forward with VMSnap but please do +be cautious. +Use this command to scrub a single bitmap from your backup disks. Keep in mind +that bitmaps are stored on a per disk basis. VMSnap will scrub each disk of the +bitmap if it find it. ```sh -vmsnap --domains="dom1" --scrub +vmsnap --domains="dom1" --scrub --scrubType=bitmap --checkpointName=virtnbdbackup.17 +``` + +To scrub a domain of **ALL** checkpoints and bitmaps + +```sh +vmsnap --domains="dom1" --scrub --scrubType=* ``` ## Contributing diff --git a/libs/general.js b/libs/general.js index 347f9a6..d60c3d3 100644 --- a/libs/general.js +++ b/libs/general.js @@ -2,14 +2,24 @@ import { access } from 'fs/promises'; import commandExists from 'command-exists'; import { ERR_DOMAINS, + ERR_INVALID_SCRUB_TYPE, ERR_REQS, ERR_SCRUB, + ERR_TOO_MANY_COMMANDS, logger, } from '../vmsnap.js'; import { cleanupCheckpoints, fetchAllDomains, VIRSH } from './virsh.js'; import { cleanupBitmaps, QEMU_IMG } from './qemu-img.js'; import { BACKUP } from './libnbdbackup.js'; +const SCRUB_TYPE_CHECKPOINT = 'checkpoint'; + +const SCRUB_TYPE_BITMAP = 'bitmap'; + +const SCRUB_TYPE_BOTH = 'both'; + +const SCRUB_TYPE_ALL = '*'; + /** * General functions used by vmsnap. * @@ -119,7 +129,11 @@ const fileExists = async (path) => { * @returns {Promise} true if the scrubbing was successful, false if * there was a failure. */ -const scrubCheckpointsAndBitmaps = async (domains) => { +const scrubCheckpointsAndBitmaps = async ({ + domains, + checkpointName, + scrubType, +}) => { if (!domains) { throw new Error('No domains specified', { code: ERR_DOMAINS }); } @@ -132,9 +146,23 @@ const scrubCheckpointsAndBitmaps = async (domains) => { for (const domain of await parseArrayParam(domains, fetchAllDomains)) { logger.info(`Scrubbing domain: ${domain}`); - await cleanupCheckpoints(domain); - - await cleanupBitmaps(domain); + if (scrubType === SCRUB_TYPE_CHECKPOINT) { + await cleanupCheckpoints(domain, checkpointName); + } else if (scrubType === SCRUB_TYPE_BITMAP) { + await cleanupBitmaps(domain, checkpointName); + } else if (scrubType === SCRUB_TYPE_BOTH) { + await cleanupCheckpoints(domain, checkpointName); + + await cleanupBitmaps(domain, checkpointName); + } else if (scrubType === '*') { + await cleanupCheckpoints(domain); + + await cleanupBitmaps(domain); + } else { + logger.error('No scrub type specified', { + code: ERR_INVALID_SCRUB_TYPE, + }); + } } scrubbed = true; diff --git a/libs/libnbdbackup.js b/libs/libnbdbackup.js index 8bdf29b..67dc914 100644 --- a/libs/libnbdbackup.js +++ b/libs/libnbdbackup.js @@ -3,6 +3,7 @@ import { spawn } from 'child_process'; import { rm } from 'fs/promises'; import dayjs from 'dayjs'; import advancedFormat from 'dayjs/plugin/advancedFormat.js'; +import quarterOfYear from 'dayjs/plugin/quarterOfYear.js'; import { logger } from '../vmsnap.js'; import { cleanupCheckpoints, domainExists, fetchAllDomains } from './virsh.js'; import { fileExists, parseArrayParam } from './general.js'; @@ -17,6 +18,7 @@ import { cleanupBitmaps } from './qemu-img.js'; // The dayjs library with the advanced format plugin. We need quarter // resolution for the backup directories. dayjs.extend(advancedFormat); +dayjs.extend(quarterOfYear); export const BACKUP = 'virtnbdbackup'; @@ -33,7 +35,7 @@ const FREQUENCY_QUARTERLY = 'quarter'; const FREQUENCY_YEARLY = 'year'; const PRUNING_FREQUENCIES = [ - FREQUENCY_MONTHLY, + FREQUENCY_MONTHLY, FREQUENCY_QUARTERLY, FREQUENCY_YEARLY, ]; @@ -44,22 +46,29 @@ const PRUNING_FREQUENCIES = [ * * @returns {string} the current months backup folder name */ -const getBackupFolder = (groupBy = FREQUENCY_MONTHLY) => { - let backupFolder; +const getBackupFolder = (groupBy = FREQUENCY_MONTHLY, current = true) => { + let lastFolder; switch (groupBy) { case FREQUENCY_QUARTERLY: - backupFolder = dayjs().format(FORMAT_QUARTERLY); + lastFolder = current + ? dayjs().format(FORMAT_QUARTERLY) + : dayjs().subtract(3, 'months').format(FORMAT_QUARTERLY); break; case FREQUENCY_YEARLY: - backupFolder = dayjs().format(FORMAT_YEARLY); + lastFolder = current + ? dayjs().format(FORMAT_YEARLY) + : dayjs().subtract(1, 'year').format(FORMAT_YEARLY); break; case FREQUENCY_MONTHLY: + lastFolder = current + ? dayjs().format(FORMAT_MONTHLY) + : dayjs().subtract(1, 'month').format(FORMAT_MONTHLY); default: - backupFolder = dayjs().format(FORMAT_MONTHLY); + return undefined; } - return `vmsnap-backup-${groupBy}ly-${backupFolder}`; + return `vmsnap-backup-${groupBy}ly-${lastFolder}`; }; /** @@ -68,13 +77,7 @@ const getBackupFolder = (groupBy = FREQUENCY_MONTHLY) => { * * @param {Object} args the command line arguments (domans, output, raw, prune) */ -const performBackup = async ({ - domains, - output, - raw, - groupBy, - prune, -}) => { +const performBackup = async ({ domains, output, raw, groupBy, prune }) => { if (!domains) { throw new Error('No domains specified', { code: ERR_DOMAINS }); } @@ -84,7 +87,7 @@ const performBackup = async ({ } for (const domain of await parseArrayParam(domains, fetchAllDomains)) { - if (await isCleanupRequired(domain, groupBy, prune, output)) { + if (await isCleanupRequired(domain, groupBy, output)) { logger.info('Creating a new backup directory, running bitmap cleanup'); await cleanupCheckpoints(domain); @@ -116,22 +119,21 @@ const performBackup = async ({ * @param {*} path the full backup directory path * @returns {Promise} true if cleanup is required, false otherwise */ -const isCleanupRequired = async (domain, groupBy, pruneFrequency, path) => { - // We're not pruning, so no cleanup is required. - if (pruneFrequency === undefined || pruneFrequency === false) { - return false; - } - - const currentBackupFolderExists = await fileExists( +const isCleanupRequired = async (domain, groupBy, path) => { + const backupFolderExists = await fileExists( `${path}${sep}${domain}${sep}${getBackupFolder(groupBy)}`, ); // Cleanup is required if the backup folder does not exist. We do this to // ensure we don't overwrite the previous months backups and to establish a // full backup for the start of the new period. - // - // If the backup folder exists, we assume the cleanup has already been done. - return currentBackupFolderExists === false; + if (!backupFolderExists) { + logger.info('Backup folder does not exist, cleanup required'); + + return true; + } + + return false; }; /** @@ -144,25 +146,28 @@ const isCleanupRequired = async (domain, groupBy, pruneFrequency, path) => { * @param {string} path the full backup directory path * @returns true if pruning is required, false otherwise */ -const isPruningRequired = async ( - domain, - groupBy, - pruneFrequency, - path, -) => { - if (pruneFrequency === undefined || pruneFrequency === false) { +const isPruningRequired = async (domain, groupBy, pruneFrequency, path) => { + if (pruneFrequency === false) { return false; // No pruning required } // If the window is not found, assume no pruning is required. if (!PRUNING_FREQUENCIES.includes(groupBy)) { - logger.warn(`Invalid prune frequency: ${groupBy}. Pruning disabled`); + logger.warn(`Invalid groupBy: ${groupBy}. Pruning disabled`); + + return false; + } + + const previousBackupFolder = getBackupFolder(groupBy, false); + + if (previousBackupFolder === undefined) { + logger.info('Unable to determine previous backup folder, skipping pruning'); return false; } const previousBackupFolderExists = await fileExists( - `${path}${sep}${domain}${sep}${getPreviousBackupFolder(groupBy)}`, + `${path}${sep}${domain}${sep}${previousBackupFolder}`, ); if (!previousBackupFolderExists) { @@ -171,9 +176,11 @@ const isPruningRequired = async ( // The number of days between the current date and the start of the backup // period. - const days = getBackupStartDate(groupBy).diff(dayjs().date(), 'days'); + const days = dayjs().diff(getBackupStartDate(groupBy), 'days'); - switch (groupBy) { + logger.info(`Days since the start of the ${groupBy}: ${days}`); + + switch (groupBy.toLowerCase()) { case FREQUENCY_MONTHLY: return days >= 15; case FREQUENCY_QUARTERLY: @@ -196,10 +203,16 @@ const isPruningRequired = async ( * @param {string} path the full backup directory path */ const pruneLastMonthsBackups = async (domain, groupBy, path) => { - const previousBackupFolder = getPreviousBackupFolder(groupBy); + const previousBackupFolder = getBackupFolder(groupBy, false); + + if (previousBackupFolder === undefined) { + logger.info('Unable to determine previous backup folder, skipping pruning'); - console.info( - `Pruning ${window} backup (${previousBackupFolder}) for ${domain}`, + return; + } + + logger.info( + `Pruning ${groupBy}ly backup (${previousBackupFolder}) for ${domain}`, ); await rm(`${path}${sep}${domain}${sep}${previousBackupFolder}`, { @@ -209,42 +222,21 @@ const pruneLastMonthsBackups = async (domain, groupBy, path) => { }; /** + * Finds and returns the start date for the backup period. * - * @param {string} groupBy the frequency to prune the backups (monthly, - * quarterly, yearly) + * @param {string} groupBy the frequency to prune the backups (month, quarter, + * year) * @returns {dayjs} the start date for the backup */ const getBackupStartDate = (groupBy) => { - switch (groupBy.toLowerCase) { - case FREQUENCY_MONTHLY: - return dayjs().startOf('month'); + switch (groupBy.toLowerCase()) { case FREQUENCY_QUARTERLY: return dayjs().startOf('quarter'); case FREQUENCY_YEARLY: return dayjs().startOf('year'); - default: - throw new Error(`Invalid prune groupBy: ${groupBy}`); - } -}; - -/** - * Returns last months backup folder name. - * - * @param {string} groupBy the frequency to prune the backups (monthly, - * quarterly, yearly) - * @returns {string} Previous month in the format of YYYY-MM or the format for - * the previous period that matches the frequency of the groupBy. - */ -const getPreviousBackupFolder = (groupBy) => { - switch (groupBy.toLowerCase) { case FREQUENCY_MONTHLY: - return dayjs().subtract(1, 'month').format(FORMAT_MONTHLY); - case FREQUENCY_QUARTERLY: - return dayjs().subtract(3, 'months').format(FORMAT_QUARTERLY); - case FREQUENCY_YEARLY: - return dayjs().subtract(1, 'year').format(FORMAT_YEARLY); default: - throw new Error(`Invalid prune frequency: ${groupBy}`); + return dayjs().startOf('month'); } }; @@ -253,12 +245,7 @@ const getPreviousBackupFolder = (groupBy) => { * * @param {Promise} domain the domain to backup */ -const backup = async ( - domain, - outputDir, - raw, - groupBy, -) => { +const backup = async (domain, outputDir, raw, groupBy) => { if (!(await domainExists(domain))) { logger.warn(`${domain} does not exist`); diff --git a/libs/qemu-img.js b/libs/qemu-img.js index db38df4..fffb20d 100644 --- a/libs/qemu-img.js +++ b/libs/qemu-img.js @@ -1,7 +1,7 @@ import { sep } from 'path'; import { asyncExec, logger } from '../vmsnap.js'; import { findKeyByValue } from './general.js'; -import { fetchAllDisks } from './virsh.js'; +import { CHECKPOINT_REGEX, fetchAllDisks } from './virsh.js'; /** * The qemu-img command interface. @@ -60,7 +60,7 @@ const findBitmaps = async (domain) => { * @param {string} domain the domain to cleanup bitmaps for bitmaps for besides * any virtual disks found. */ -const cleanupBitmaps = async (domain) => { +const cleanupBitmaps = async (domain, checkpointName = undefined) => { const bitmaps = await findBitmaps(domain); for (const record of bitmaps) { @@ -73,7 +73,12 @@ const cleanupBitmaps = async (domain) => { for (const bitmap of record.bitmaps) { // Adding just in case we have a bitmap that isn't ours. Not sure if // this is possible, but better safe than sorry. - if (/^virtnbdbackup\.[0-9]*$/.test(bitmap.name) === false) { + if (CHECKPOINT_REGEX.test(bitmap.name) === false) { + continue; + } + + // If we have a checkpoint name and it doesn't match, skip it + if (checkpointName && bitmap.name !== checkpointName) { continue; } @@ -86,14 +91,10 @@ const cleanupBitmaps = async (domain) => { ]; logger.info( - `- Removing bitmap ${bitmap.name} from ${record.path} on ${domain}`, + `Removing bitmap ${bitmap.name} from ${record.path} on ${domain}`, ); try { - logger.info( - `- Removing bitmap ${bitmap.name} from ${record.path} on ${domain}`, - ); - await asyncExec(command.join(' ')); } catch (error) { logger.warn( diff --git a/libs/virsh.js b/libs/virsh.js index 327e0fc..7896e43 100644 --- a/libs/virsh.js +++ b/libs/virsh.js @@ -10,6 +10,8 @@ import { logger } from '../vmsnap.js'; export const VIRSH = 'virsh'; +export const CHECKPOINT_REGEX = /^virtnbdbackup\.[0-9]*$/; + /** * Check if a domain exists on the host system. * @@ -76,7 +78,7 @@ const findCheckpoints = async (domain) => { * * @param {string} domain the domain to cleanup checkpoints for */ -const cleanupCheckpoints = async (domain) => { +const cleanupCheckpoints = async (domain, checkpointName = undefined) => { const checkpoints = await findCheckpoints(domain); if (checkpoints.length === 0) { @@ -86,8 +88,13 @@ const cleanupCheckpoints = async (domain) => { for (const checkpoint of checkpoints) { // Adding just in case we have a checkpoint that isn't ours. Not sure if // this is possible, but better safe than sorry. - if (/^virtnbdbackup\.[0-9]*$/.test(checkpoint) === false) { + if (CHECKPOINT_REGEX.test(checkpoint) === false) { continue; + } + + // If we have a checkpoint name and it doesn't match, skip it + if (checkpointName && checkpoint !== checkpointName) { + continue } const command = [ diff --git a/package-lock.json b/package-lock.json index 2edc189..0b8a2a7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "vmsnap", - "version": "1.0.6-alpha", + "version": "1.1.0-alpha", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "vmsnap", - "version": "1.0.6-alpha", + "version": "1.1.0-alpha", "license": "MIT", "dependencies": { "chalk": "^5.3.0", diff --git a/package.json b/package.json index ee505aa..06d345e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "vmsnap", - "version": "1.0.6-alpha", + "version": "1.1.0-alpha", "description": "A Node based backup and backup rotation tool for KVM domains.", "main": "dist/vmsnap.js", "scripts": { @@ -9,6 +9,7 @@ "lint": "npx eslint *.js", "check-format": "npx prettier --check \"*.js\"", "format": "npx prettier --write \"*.js\"", + "watch": "rimraf dist && esbuild --minify --platform=node --outdir=dist vmsnap.js libs/*.js --watch", "build": "rimraf dist && esbuild --minify --platform=node --outdir=dist vmsnap.js libs/*.js", "prepare": "husky install || true" }, @@ -65,6 +66,7 @@ "keywords": [ "kvm", "backup", + "incremental", "snapshot", "rotation", "qemu", diff --git a/vmsnap.js b/vmsnap.js index 1a73699..0cef54b 100755 --- a/vmsnap.js +++ b/vmsnap.js @@ -59,6 +59,9 @@ export const ERR_LOCK_RELEASE = 6; // More than one command was specified. export const ERR_TOO_MANY_COMMANDS = 7; +// Invalid scrub type was specified. +export const ERR_INVALID_SCRUB_TYPE = 8; + // A spinnner for long running tasks export const spinner = yoctoSpinner(); @@ -133,7 +136,7 @@ lock(lockfile, { retries: 10, retryWait: 10000 }, async () => { checkCommand(argv); if (argv.scrub) { - await scrubCheckpointsAndBitmaps(argv.domains); + await scrubCheckpointsAndBitmaps(argv); } else if (argv.backup) { await performBackup(argv); } else {