From a3d378a972b1abf290aa36d93272828ea3993901 Mon Sep 17 00:00:00 2001 From: arran Date: Sat, 23 Jul 2022 23:08:11 +0800 Subject: [PATCH] add templates and scripts for libvirt provider, tidy --- .gitignore | 13 ++- CHANGELOG.md | 7 ++ HACKING.md | 22 ++++ README.md | 25 +++- libvirt/.gitignore | 1 + libvirt/Makefile | 42 +++++++ libvirt/ubuntu_box.pkr.hcl | 109 ++++++++++++++++++ .../make_templates.pl => make_templates.pl | 0 provision-01.sh | 50 ++++++++ provision-02.sh | 45 ++++++++ virtualbox/publish.sh => publish.sh | 8 +- rules.mk | 69 +++++++++++ virtualbox/vars.mk => vars.mk | 2 - virtualbox/.gitignore | 1 + virtualbox/Makefile | 55 ++------- virtualbox/ubuntu_box.pkr.hcl | 10 +- 16 files changed, 391 insertions(+), 68 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 HACKING.md create mode 100644 libvirt/.gitignore create mode 100644 libvirt/Makefile create mode 100644 libvirt/ubuntu_box.pkr.hcl rename virtualbox/make_templates.pl => make_templates.pl (100%) create mode 100755 provision-01.sh create mode 100755 provision-02.sh rename virtualbox/publish.sh => publish.sh (97%) create mode 100644 rules.mk rename virtualbox/vars.mk => vars.mk (77%) create mode 100644 virtualbox/.gitignore diff --git a/.gitignore b/.gitignore index 26ce2cf..430ea2a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,14 @@ .DS_Store .*.swp -_site/* -/config -/out/ -node_modules -/build/ +build/ TOKEN .release* .upload* -/ova/ +ova/ developer.rb info.json +.box* +.provider* +.version* +*XXX* +*xxx* diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e20bb2e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog for cits3007 vagrant environment + +## Unreleased changes + +## 0.1.0 - 2021-07-24 + +Initial version diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 0000000..1666a20 --- /dev/null +++ b/HACKING.md @@ -0,0 +1,22 @@ + +# useful tips + +## VMDK info + +A handy command for displaying info about a .vmdk file is: + +``` +vboxmanage showmediuminfo /path/to/img.vmdk +``` + +It shows the file format version, and what disk variant is being used (streaming, fixed, etc.) + +## Building an .ova file + +It would be nice if we could just download an OVA file from the Vagrant cloud, +but it appears that the generic/ubuntu2004 box consists of a version 1 OVA file, +but packer only works with version 2 OVA files. So we have to instead re-package +the OVA file using `make_ova.pl` (which used as a template the output of doing an +"export" from a recent version of VirtualBox). + + diff --git a/README.md b/README.md index a424abc..0afe4a4 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,26 @@ # cits3007 ubuntu-vagrant-box Packer scripts for building Ubuntu 20.04-based Vagrant boxes. +Published boxes are hosted at +. -Requires Hashicorp Vagrant and an appropriate box provider (e.g. -either libvirt or VirtualBox) be installed, plus common Unix -commands. +## Build instructions -`make packer-build` will build a box ready for uploading to the -Vagrant cloud, and `make publish` will publish it. +Requires that Hashicorp Packer, Hashicorp Vagrant, and an appropriate box provider +(e.g. either libvirt or VirtualBox) be installed, plus: -## Usage +- GNU make +- jq + +`cd` into either the `virtualbox` or `libvirt` directories. +Then `make packer-build` will build a box ready for uploading to the Vagrant +cloud. + +`make publish` will publish it; this requires you have a file called TOKEN +in the relevant directory (it can be a symlink), containing a Vagrant cloud +access token. + +## Box usage To use and `ssh` into a VirtualBox-based box: @@ -19,4 +30,6 @@ $ vagrant up --provider=virtualbox $ vagrant ssh ``` +For a libvirt-based box, do the same but use `--provider=libvirt`. + diff --git a/libvirt/.gitignore b/libvirt/.gitignore new file mode 100644 index 0000000..485d449 --- /dev/null +++ b/libvirt/.gitignore @@ -0,0 +1 @@ +.qcow_checksum.md5 diff --git a/libvirt/Makefile b/libvirt/Makefile new file mode 100644 index 0000000..ec2c142 --- /dev/null +++ b/libvirt/Makefile @@ -0,0 +1,42 @@ + +include ../vars.mk + +PROVIDER=libvirt + +include ../rules.mk + + + +QCOW_PATH=$(HOME)/.vagrant.d/boxes/generic-VAGRANTSLASH-$(BASE_BOX_NAME)/$(BASE_BOX_VERSION)/$(PROVIDER)/box.img + +$(QCOW_PATH): + vagrant box add \ + --provider $(PROVIDER) \ + --box-version $(BASE_BOX_VERSION) \ + $(BASE_BOX) + +.qcow_checksum.md5: $(QCOW_PATH) + cat $(QCOW_PATH) | pv | md5sum | awk '{ print $$1; }' > $@ + +build/$(BOX_NAME)_$(BOX_VERSION).box \ +build/$(BOX_NAME)_$(BOX_VERSION).box.md5 \ +build/$(BOX_NAME)_$(BOX_VERSION).qcow2 \ + : $(QCOW_PATH) \ + developer.rb \ + info.json \ + .qcow_checksum.md5 + set -x && \ + export PKR_VAR_SOURCE_PATH=$< && \ + export PKR_VAR_SOURCE_MD5=`cat .qcow_checksum.md5` && \ + export PKR_VAR_OUTPUT_DIR=$(output_dir) && \ + export PKR_VAR_DISK_SIZE=`qemu-img info --output=json $(QCOW_PATH) | jq '.["virtual-size"]'` && \ + export PKR_VAR_BOX_NAME=$(BOX_NAME) && \ + export PKR_VAR_BOX_VERSION=$(BOX_VERSION) && \ + packer validate $(packer_template) && \ + PACKER_LOG=1 packer build $(packer_template) + +packer-build: build/$(BOX_NAME)_$(BOX_VERSION).box + + + + diff --git a/libvirt/ubuntu_box.pkr.hcl b/libvirt/ubuntu_box.pkr.hcl new file mode 100644 index 0000000..18bc990 --- /dev/null +++ b/libvirt/ubuntu_box.pkr.hcl @@ -0,0 +1,109 @@ + +# various variables, typically +# got from environment + +variable "SOURCE_PATH" { + type = string + description = "path to input .img QCOW2 file" +} + +variable "SOURCE_MD5" { + type = string + description = "md5 hash of QCOW2 file" +} + +variable "DISK_SIZE" { + type = number + description = "size of disk in bytes" +} + +variable "OUTPUT_DIR" { + type = string + description = "output dir" +} + +variable "BOX_NAME" { + type = string + description = "name of box being created" +} + +variable "BOX_VERSION" { + type = string + description = "version of box being created" +} + +source "qemu" "cits3007" { + + iso_url = "file:///${var.SOURCE_PATH}" + disk_image = true + format = "qcow2" + iso_checksum = "md5:${var.SOURCE_MD5}" + + shutdown_command = "sudo shutdown -P now" + + communicator = "ssh" + ssh_username = "vagrant" + ssh_password = "vagrant" + ssh_timeout = "60m" + + headless = true + + # Needn't specify an accelerator - packer docco + # says kvm will be used by default if available, + # else tcg: https://www.packer.io/docs/builders/qemu. + + #accelerator = "kvm" + + output_directory = "${var.OUTPUT_DIR}" + + disk_size = "${var.DISK_SIZE}b" + vm_name = "${var.BOX_NAME}_${var.BOX_VERSION}.qcow2" + + net_device = "virtio-net" + disk_interface = "virtio-scsi" + boot_wait = "20s" + + display = "none" + + # needed, see https://github.com/hashicorp/packer/issues/8693 + # (??still) + qemuargs = [ + ["-display", "none"] + ] +} + + +build { + sources = ["source.qemu.cits3007"] + + provisioner "shell" { + scripts = [ + "../provision-01.sh", + "../provision-02.sh" + ] + execute_command = "sudo sh -c '{{ .Vars }} {{ .Path }}'" + timeout = "60m" + max_retries = 2 + } + + post-processors { + + post-processor "vagrant" { + + compression_level = 9 + keep_input_artifact = true + vagrantfile_template = "developer.rb" + output = "${var.OUTPUT_DIR}/${var.BOX_NAME}_${var.BOX_VERSION}.box" + include = [ + "info.json" + ] + } + + post-processor "checksum" { + checksum_types = ["md5"] + output = "${var.OUTPUT_DIR}/${var.BOX_NAME}_${var.BOX_VERSION}.box.md5" + } + + } + +} diff --git a/virtualbox/make_templates.pl b/make_templates.pl similarity index 100% rename from virtualbox/make_templates.pl rename to make_templates.pl diff --git a/provision-01.sh b/provision-01.sh new file mode 100755 index 0000000..e0d6009 --- /dev/null +++ b/provision-01.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# should be run as root + +set -x + +apt-get update + +# should already have: wget, curl + +# basic apps + +DEBIAN_FRONTEND=noninteractive \ + apt-get install --no-install-recommends -y \ + apt-transport-https \ + aptitude \ + bash \ + bash-completion \ + bzip2 \ + ca-certificates \ + command-not-found \ + expect \ + file \ + fakeroot \ + gpg \ + jq \ + less \ + lsof \ + lynx \ + netcat-openbsd \ + procps \ + pv \ + openssh-client \ + screen \ + sudo \ + time + +# extra utils + +DEBIAN_FRONTEND=noninteractive \ + apt-get install --no-install-recommends -y \ + binutils \ + bsdmainutils \ + coreutils \ + diffutils \ + findutils \ + moreutils \ + patchutils \ + sharutils + diff --git a/provision-02.sh b/provision-02.sh new file mode 100755 index 0000000..627abd4 --- /dev/null +++ b/provision-02.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# should be run as root + +set -x + +apt-get update + +# should already have: wget, curl + + +# dev tools + +DEBIAN_FRONTEND=noninteractive \ + apt-get install --no-install-recommends -y \ + afl++-clang \ + build-essential \ + clang \ + clang-format \ + clang-tidy \ + clang-tools \ + g++-multilib \ + gdb \ + git \ + gpg \ + indent \ + libtool \ + llvm-10-dev \ + pkg-config \ + splint \ + universal-ctags \ + valgrind \ + xxd \ + zzuf + +# docco + +DEBIAN_FRONTEND=noninteractive \ + apt-get install --no-install-recommends -y \ + man-db \ + manpages \ + manpages-dev \ + manpages-posix \ + manpages-posix-dev + diff --git a/virtualbox/publish.sh b/publish.sh similarity index 97% rename from virtualbox/publish.sh rename to publish.sh index 880f324..14f58e8 100755 --- a/virtualbox/publish.sh +++ b/publish.sh @@ -12,13 +12,14 @@ # $ sudo add-apt-repository ppa:savoury1/curl34 # $ sudo apt-get install curl -if (( $# != 2 )); then - echo >&2 "expected 2 args: PATH_TO_BOX PATH_TO_MD5" +if (( $# != 3 )); then + echo >&2 "expected 3 args: PATH_TO_BOX PATH_TO_MD5 PROVIDER" exit 1 fi BOX_PATH="$1"; shift BOX_MD5="$(cat $1 | awk '{print $1; }')"; shift +PROVIDER_TYPE="$1"; shift setup_colors() { if [[ -t 2 ]] && [[ -z "${NO_COLOR-}" ]] && [[ "${TERM-}" != "dumb" ]]; then @@ -48,7 +49,6 @@ fi set -uo pipefail BOX_NAME="$(make $MAKE_ARGS print_box_name)" -PROVIDER_TYPE="virtualbox" SHORT_DESC="$(make $MAKE_ARGS print_short_desc)" VAGRANT_CLOUD_USERNAME="$(make $MAKE_ARGS print_vagrant_cloud_username)" @@ -127,7 +127,7 @@ printf '\n'"${GREEN}%s${NOFORMAT}"'\n' "ensuring vagrant-cloud box named $VAGRAN cat > .provider_metadata <