diff --git a/Changelog b/Changelog index e781412d..2368f3d2 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,12 @@ +Version 2.3 +--------- + * Add option -S (--start-domain): with this option it is possible + to put an offline domain into paused state: CPU for domain is in halted state + (not executing any code), but it is possible to execute full/inc/diff + backups and create checkpoints. Domain is destroyed automatically after + backup finishes. (#164) + + Version 2.2 --------- * Fix Progressbar during restore: wrong values used. (#160) @@ -8,6 +17,10 @@ Version 2.2 condition (#163) * Pass pidFile to qemu-nbd process for local NBD server during restore, report PID of forked process instead of parent. + * Add option -S (--start-domain): if specified and virtual domain is offline + during backup, domain will be started in pause mode, allowing to execute + full/diff/inc backups. Domain is destroyed as soon as operation finished + by using libvirt's AUTODESTROY flag. (#164) Version 2.1 --------- diff --git a/README.md b/README.md index f6531a4c..adf7952b 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ of your `kvm/qemu` virtual machines. - [Supported disk formats / raw disks](#supported-disk-formats--raw-disks) - [Backup Examples](#backup-examples) - [Local full/incremental backup](#local-fullincremental-backup) + - [Backing up offline virtual domains](#backing-up-offline-virtual-domains) - [Application consistent backups](#application-consistent-backups) - [Rotating backups](#rotating-backups) - [Excluding disks](#excluding-disks) @@ -263,13 +264,6 @@ listening on a local unix socket. This nbd backend provides consistent access to the virtual machines, disk data and dirty blocks. After the backup process finishes, the job is stopped and the nbd server quits operation. -`Note`: -> If the virtual domain is not in running state (powered off) `virtnbdbackup` -> supports `copy` and `inc/diff` backup modes. Incremental and differential backups -> will then save the changed blocks since last created checkpoint. As no new -> checkpoints can be defined for offline domains, the Backup mode `full` is -> changed to mode `copy`. - It is possible to backup multiple virtual machines on the same host system at the same time, using separate calls to the application with a different target directory to store the data. @@ -340,6 +334,22 @@ backup issues: ├── vmconfig.virtnbdbackup.1.xml ``` +## Backing up offline virtual domains + +If the virtual domain is not in running state (powered off) `virtnbdbackup` +supports `copy` and `inc/diff` backup modes. Incremental and differential +backups will then save the changed blocks since last created checkpoint. As no +new checkpoints can be defined for offline domains. + +Backup mode `full` is changed to mode `copy`. + +This behavior can be changed using the `-S` (`--start-domain`) option: prior to +executing the backup, the virtual domain will then be started in `paused` +state: The virtual machines CPU's are halted, but the running QEMU Process will +allow all operations required to execute backups. + +As the backup process is finished, the domain will be destroyed automatically. + ## Application consistent backups During backup `virtnbdbackup` attempts to freeze all file systems within the diff --git a/libvirtnbdbackup/__init__.py b/libvirtnbdbackup/__init__.py index 7e9e4ab8..430055ff 100644 --- a/libvirtnbdbackup/__init__.py +++ b/libvirtnbdbackup/__init__.py @@ -15,4 +15,4 @@ along with this program. If not, see . """ -__version__ = "2.2" +__version__ = "2.3" diff --git a/libvirtnbdbackup/backup/check.py b/libvirtnbdbackup/backup/check.py new file mode 100644 index 00000000..fd795e45 --- /dev/null +++ b/libvirtnbdbackup/backup/check.py @@ -0,0 +1,109 @@ +""" + Copyright (C) 2024 Michael Ablassmeier + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +""" + +import logging +from argparse import Namespace +from libvirt import virDomain +from libvirtnbdbackup import virt +from libvirtnbdbackup import common as lib +from libvirtnbdbackup import exceptions + +log = logging.getLogger() + + +def arguments(args: Namespace) -> None: + """Check passed arguments vor validity""" + if args.compress is not False and args.type == "raw": + raise exceptions.BackupException("Compression not supported with raw output.") + + if args.stdout is True and args.type == "raw": + raise exceptions.BackupException("Output type raw not supported to stdout.") + + if args.stdout is True and args.raw is True: + raise exceptions.BackupException( + "Saving raw images to stdout is not supported." + ) + + if args.type == "raw" and args.level in ("inc", "diff"): + raise exceptions.BackupException( + "Stream format raw does not support incremental or differential backup." + ) + + +def targetDir(args: Namespace) -> None: + """Check if target directory backup is started to meets + all requirements based on the backup level executed""" + if ( + args.level not in ("copy", "full", "auto") + and not lib.hasFullBackup(args) + and not args.stdout + ): + raise exceptions.BackupException( + f"Unable to execute [{args.level}] backup: " + f"No full backup found in target directory: [{args.output}]" + ) + + if lib.targetIsEmpty(args) and args.level == "auto": + log.info("Backup mode auto, target folder is empty: executing full backup.") + args.level = "full" + elif not lib.targetIsEmpty(args) and args.level == "auto": + if not lib.hasFullBackup(args): + raise exceptions.BackupException( + "Can't execute switch to auto incremental backup: " + f"specified target folder [{args.output}] does not contain full backup.", + ) + log.info("Backup mode auto: executing incremental backup.") + args.level = "inc" + elif not args.stdout and not args.startonly and not args.killonly: + if not lib.targetIsEmpty(args): + raise exceptions.BackupException( + "Target directory already contains full or copy backup." + ) + + +def vmstate(args, virtClient: virt.client, domObj: virDomain) -> None: + """Check virtual machine state before executing backup + and based on situation, either fallback to regular copy + backup or attempt to bring VM into paused state""" + if domObj.isActive() == 0: + args.offline = True + if args.start_domain is True: + log.info("Starting domain in paused state") + if virtClient.startDomain(domObj) == 0: + args.offline = False + else: + log.info("Failed to start VM in paused mode.") + + if args.level == "full" and args.offline is True: + log.warning("Domain is offline, resetting backup options.") + args.level = "copy" + log.warning("New Backup level: [%s].", args.level) + args.offline = True + + if args.offline is True and args.startonly is True: + raise exceptions.BackupException( + "Domain is offline: must be active for this function." + ) + + +def vmfeature(virtClient: virt.client, domObj: virDomain) -> None: + """Check if required features are enabled in domain config""" + if virtClient.hasIncrementalEnabled(domObj) is False: + raise exceptions.BackupException( + "Virtual machine does not support required backup features, " + "please adjust virtual machine configuration." + ) diff --git a/libvirtnbdbackup/virt/client.py b/libvirtnbdbackup/virt/client.py index 801b340f..1b956832 100644 --- a/libvirtnbdbackup/virt/client.py +++ b/libvirtnbdbackup/virt/client.py @@ -226,6 +226,13 @@ def getDomainConfig(domObj: libvirt.virDomain) -> str: """Return Virtual Machine configuration as XML""" return domObj.XMLDesc(0) + @staticmethod + def startDomain(domObj: libvirt.virDomain) -> bool: + """Start virtual machine in paused state to allow full / inc backup""" + return domObj.createWithFlags( + flags=libvirt.VIR_DOMAIN_START_PAUSED | libvirt.VIR_DOMAIN_START_AUTODESTROY + ) + @staticmethod def domainAutoStart(domObj: libvirt.virDomain) -> None: """Mark virtual machine for autostart""" diff --git a/man/virtnbdbackup.1 b/man/virtnbdbackup.1 index 3326a908..828626ca 100644 --- a/man/virtnbdbackup.1 +++ b/man/virtnbdbackup.1 @@ -1,17 +1,17 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. -.TH VIRTNBDBACKUP "1" "March 2024" "virtnbdbackup 2.2" "User Commands" +.TH VIRTNBDBACKUP "1" "March 2024" "virtnbdbackup 2.3" "User Commands" .SH NAME virtnbdbackup \- backup utility for libvirt .SH DESCRIPTION usage: virtnbdbackup [\-h] \fB\-d\fR DOMAIN [\-l {copy,full,inc,diff,auto}] .TP [\-t {stream,raw}] [\-r] \fB\-o\fR OUTPUT [\-C CHECKPOINTDIR] -[\-S SCRATCHDIR] [\-i INCLUDE] [\-x EXCLUDE] [\-f SOCKETFILE] -[\-n] [\-z [COMPRESS]] [\-w WORKER] [\-F FREEZE_MOUNTPOINT] -[\-e] [\-T THRESHOLD] [\-U URI] [\-\-user USER] -[\-\-ssh\-user SSH_USER] [\-\-password PASSWORD] [\-P NBD_PORT] -[\-I NBD_IP] [\-\-tls] [\-\-tls\-cert TLS_CERT] [\-L] [\-\-quiet] -[\-\-nocolor] [\-q] [\-s] [\-k] [\-p] [\-v] [\-V] +[\-\-scratchdir SCRATCHDIR] [\-S] [\-i INCLUDE] [\-x EXCLUDE] +[\-f SOCKETFILE] [\-n] [\-z [COMPRESS]] [\-w WORKER] +[\-F FREEZE_MOUNTPOINT] [\-e] [\-T THRESHOLD] [\-U URI] +[\-\-user USER] [\-\-ssh\-user SSH_USER] [\-\-password PASSWORD] +[\-P NBD_PORT] [\-I NBD_IP] [\-\-tls] [\-\-tls\-cert TLS_CERT] +[\-L] [\-\-quiet] [\-\-nocolor] [\-q] [\-s] [\-k] [\-p] [\-v] [\-V] .PP Backup libvirt/qemu virtual machines .SS "options:" @@ -38,9 +38,12 @@ Output target directory \fB\-C\fR CHECKPOINTDIR, \fB\-\-checkpointdir\fR CHECKPOINTDIR Persistent libvirt checkpoint storage directory .TP -\fB\-S\fR SCRATCHDIR, \fB\-\-scratchdir\fR SCRATCHDIR +\fB\-\-scratchdir\fR SCRATCHDIR Target dir for temporary scratch file. (default: \fI\,/var/tmp\/\fP) .TP +\fB\-S\fR, \fB\-\-start\-domain\fR +Start virtual machine if it is offline. (default: False) +.TP \fB\-i\fR INCLUDE, \fB\-\-include\fR INCLUDE Backup only disk with target dev name (\fB\-i\fR vda) .TP @@ -48,7 +51,7 @@ Backup only disk with target dev name (\fB\-i\fR vda) Exclude disk(s) with target dev name (\fB\-x\fR vda,vdb) .TP \fB\-f\fR SOCKETFILE, \fB\-\-socketfile\fR SOCKETFILE -Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.2487158\/\fP) +Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.37868\/\fP) .TP \fB\-n\fR, \fB\-\-noprogress\fR Disable progress bar diff --git a/man/virtnbdmap.1 b/man/virtnbdmap.1 index 1bced5b4..3a7dcc76 100644 --- a/man/virtnbdmap.1 +++ b/man/virtnbdmap.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. -.TH VIRTNBDMAP "1" "March 2024" "virtnbdmap 2.2" "User Commands" +.TH VIRTNBDMAP "1" "March 2024" "virtnbdmap 2.3" "User Commands" .SH NAME virtnbdmap \- map virtnbdbackup image files to nbd devices .SH DESCRIPTION diff --git a/man/virtnbdrestore.1 b/man/virtnbdrestore.1 index f29a15eb..de4f84a0 100644 --- a/man/virtnbdrestore.1 +++ b/man/virtnbdrestore.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. -.TH VIRTNBDRESTORE "1" "March 2024" "virtnbdrestore 2.2" "User Commands" +.TH VIRTNBDRESTORE "1" "March 2024" "virtnbdrestore 2.3" "User Commands" .SH NAME virtnbdrestore \- restore utility for libvirt .SH DESCRIPTION @@ -40,7 +40,7 @@ Process only disk matching target dev name. (default: None) Disable progress bar .TP \fB\-f\fR SOCKETFILE, \fB\-\-socketfile\fR SOCKETFILE -Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.2487163\/\fP) +Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.37873\/\fP) .TP \fB\-r\fR, \fB\-\-raw\fR Copy raw images as is during restore. (default: False) diff --git a/t/tests.bats b/t/tests.bats index f4f15e35..1e8889fb 100644 --- a/t/tests.bats +++ b/t/tests.bats @@ -503,6 +503,16 @@ toOut() { [ "$status" -eq 0 ] run virsh start $VM } +@test "Offline Backup: full backup with vm startup option" { + run virsh destroy $VM + run ../virtnbdbackup -d $VM -l full -o ${TMPDIR}/offline-full-option -S + echo "output = ${output}" + [ "$status" -eq 0 ] + [[ "${output}" =~ "Starting domain in paused state" ]] + run virsh start $VM + [ "$status" -eq 0 ] +} + @test "Restore: restore vm and adjust vm config" { [ -z $INCTEST ] && skip "skipping" rm -rf ${TMPDIR}/RESTORECONFIG/ diff --git a/virtnbdbackup b/virtnbdbackup index da06c17d..f841568b 100755 --- a/virtnbdbackup +++ b/virtnbdbackup @@ -39,6 +39,7 @@ from libvirtnbdbackup.backup import partialfile from libvirtnbdbackup.backup import job from libvirtnbdbackup.backup import disk from libvirtnbdbackup.backup import metadata +from libvirtnbdbackup.backup import check from libvirtnbdbackup.ssh.exceptions import sshError from libvirtnbdbackup.virt.exceptions import ( domainNotFound, @@ -111,13 +112,20 @@ def main() -> None: help="Persistent libvirt checkpoint storage directory", ) opt.add_argument( - "-S", "--scratchdir", default="/var/tmp", required=False, type=str, help="Target dir for temporary scratch file. (default: %(default)s)", ) + opt.add_argument( + "-S", + "--start-domain", + default=False, + required=False, + action="store_true", + help="Start virtual machine if it is offline. (default: %(default)s)", + ) opt.add_argument( "-i", "--include", @@ -277,48 +285,6 @@ def main() -> None: if args.compress is not False: logging.info("Compression enabled, level [%s]", args.compress) - if args.compress is not False and args.type == "raw": - logging.error("Compression not supported with raw output.") - sys.exit(1) - - if args.stdout is True and args.type == "raw": - logging.error("Output type raw not supported to stdout.") - sys.exit(1) - - if args.stdout is True and args.raw is True: - logging.error("Saving raw images to stdout is not supported.") - sys.exit(1) - - if ( - args.level not in ("copy", "full", "auto") - and not lib.hasFullBackup(args) - and not args.stdout - ): - logging.error( - "Unable to execute [%s] backup: No full backup found in target directory: [%s]", - args.level, - args.output, - ) - sys.exit(1) - - if lib.targetIsEmpty(args) and args.level == "auto": - logging.info("Backup mode auto, target folder is empty: executing full backup.") - args.level = "full" - elif not lib.targetIsEmpty(args) and args.level == "auto": - if not lib.hasFullBackup(args): - logging.error( - "Can't execute switch to auto incremental backup: " - "specified target folder [%s] does not contain full backup.", - args.output, - ) - sys.exit(1) - logging.info("Backup mode auto: executing incremental backup.") - args.level = "inc" - elif not args.stdout and not args.startonly and not args.killonly: - if not lib.targetIsEmpty(args): - logging.error("Target directory already contains full or copy backup.") - sys.exit(1) - if args.raw is True and args.level in ("inc", "diff"): logging.warning( "Raw disks can't be included during incremental or differential backup." @@ -326,10 +292,10 @@ def main() -> None: logging.warning("Excluding raw disks.") args.raw = False - if args.type == "raw" and args.level in ("inc", "diff"): - logging.error( - "Stream format raw does not support incremental or differential backup." - ) + try: + check.arguments(args) + except exceptions.BackupException as e: + logging.error(e) sys.exit(1) if partialfile.exists(args): @@ -354,30 +320,17 @@ def main() -> None: logging.info("Libvirt library version: [%s]", virtClient.libvirtVersion) - if virtClient.hasIncrementalEnabled(domObj) is False: - logging.error( - ( - "Virtual machine does not support required backup features, " - "please adjust virtual machine configuration." - ) - ) - sys.exit(1) - try: checkpoint.checkForeign(args, domObj) except exceptions.CheckpointException: sys.exit(1) - if domObj.isActive() == 0: - if args.level == "full": - logging.warning("Domain is offline, resetting backup options.") - args.level = "copy" - logging.warning("New Backup level: [%s].", args.level) - args.offline = True - - if args.offline is True and args.startonly is True: - logging.error("Virtual machine is currently offline") - logging.error("Virtual machine must be active for this function.") + try: + check.vmfeature(virtClient, domObj) + check.vmstate(args, virtClient, domObj) + check.targetDir(args) + except exceptions.BackupException as e: + logging.error(e) sys.exit(1) signal.signal( @@ -397,6 +350,7 @@ def main() -> None: logging.error("Unable to detect disks suitable for backup.") metadata.saveFiles(args, vmConfig, disks, fileStream, logFile) sys.exit(1) + if ( not args.killonly and not args.offline