add templates and scripts for libvirt provider, tidy

.gitignore

@@ -1,13 +1,14 @@
 .DS_Store
 .*.swp
-_site/*
-/config
-/out/
-node_modules
-/build/
+build/
 TOKEN
 .release*
 .upload*
-/ova/
+ova/
 developer.rb
 info.json
+.box*
+.provider*
+.version*
+*XXX*
+*xxx*

CHANGELOG.md

@@ -0,0 +1,7 @@
+# Changelog for cits3007 vagrant environment
+
+## Unreleased changes
+
+## 0.1.0 - 2021-07-24
+
+Initial version

HACKING.md

@@ -0,0 +1,22 @@
+
+# useful tips
+
+## VMDK info
+
+A handy command for displaying info about a .vmdk file is:
+
+```
+vboxmanage showmediuminfo /path/to/img.vmdk
+```
+
+It shows the file format version, and what disk variant is being used (streaming, fixed, etc.)
+
+## Building an .ova file
+
+It would be nice if we could just download an OVA file from the Vagrant cloud,
+but it appears that the generic/ubuntu2004 box consists of a version 1 OVA file,
+but packer only works with version 2 OVA files. So we have to instead re-package
+the OVA file using `make_ova.pl` (which used as a template the output of doing an
+"export" from a recent version of VirtualBox).
+
+

README.md

@@ -1,15 +1,26 @@
 # cits3007 ubuntu-vagrant-box
 
 Packer scripts for building Ubuntu 20.04-based Vagrant boxes.
+Published boxes are hosted at
+<https://app.vagrantup.com/arranstewart/boxes/cits3007-ubuntu2004>.
 
-Requires Hashicorp Vagrant and an appropriate box provider (e.g.
-either libvirt or VirtualBox) be installed, plus common Unix
-commands.
+## Build instructions
 
-`make packer-build` will build a box ready for uploading to the
-Vagrant cloud, and `make publish` will publish it.
+Requires that Hashicorp Packer, Hashicorp Vagrant, and an appropriate box provider
+(e.g.  either libvirt or VirtualBox) be installed, plus:
 
-## Usage
+- GNU make
+- jq
+
+`cd` into either the `virtualbox` or `libvirt` directories.
+Then `make packer-build` will build a box ready for uploading to the Vagrant
+cloud.
+
+`make publish` will publish it; this requires you have a file called TOKEN
+in the relevant directory (it can be a symlink), containing a Vagrant cloud
+access token.
+
+## Box usage
 
 To use and `ssh` into a VirtualBox-based box:
 
@@ -19,4 +30,6 @@ $ vagrant up --provider=virtualbox
 $ vagrant ssh
 ```
 
+For a libvirt-based box, do the same but use `--provider=libvirt`.
+
 

libvirt/.gitignore

@@ -0,0 +1 @@
+.qcow_checksum.md5

libvirt/Makefile

@@ -0,0 +1,42 @@
+
+include ../vars.mk
+
+PROVIDER=libvirt
+
+include ../rules.mk
+
+
+
+QCOW_PATH=$(HOME)/.vagrant.d/boxes/generic-VAGRANTSLASH-$(BASE_BOX_NAME)/$(BASE_BOX_VERSION)/$(PROVIDER)/box.img
+
+$(QCOW_PATH):
+	vagrant box add \
+	    --provider $(PROVIDER) \
+	    --box-version $(BASE_BOX_VERSION) \
+	    $(BASE_BOX)
+
+.qcow_checksum.md5: $(QCOW_PATH)
+	cat $(QCOW_PATH) | pv | md5sum | awk '{ print $$1; }' > $@
+
+build/$(BOX_NAME)_$(BOX_VERSION).box \
+build/$(BOX_NAME)_$(BOX_VERSION).box.md5 \
+build/$(BOX_NAME)_$(BOX_VERSION).qcow2 \
+		: $(QCOW_PATH) \
+			developer.rb \
+			info.json \
+			.qcow_checksum.md5
+	set -x && \
+	export PKR_VAR_SOURCE_PATH=$< && \
+	export PKR_VAR_SOURCE_MD5=`cat .qcow_checksum.md5` && \
+	export PKR_VAR_OUTPUT_DIR=$(output_dir) && \
+	export PKR_VAR_DISK_SIZE=`qemu-img info --output=json $(QCOW_PATH) | jq '.["virtual-size"]'` && \
+	export PKR_VAR_BOX_NAME=$(BOX_NAME) && \
+	export PKR_VAR_BOX_VERSION=$(BOX_VERSION) && \
+	packer validate $(packer_template) && \
+	PACKER_LOG=1 packer build $(packer_template)
+
+packer-build: build/$(BOX_NAME)_$(BOX_VERSION).box
+
+
+
+

libvirt/ubuntu_box.pkr.hcl

@@ -0,0 +1,109 @@
+
+# various variables, typically
+# got from environment
+
+variable "SOURCE_PATH" {
+  type        = string
+  description = "path to input .img QCOW2 file"
+}
+
+variable "SOURCE_MD5" {
+  type        = string
+  description = "md5 hash of QCOW2 file"
+}
+
+variable "DISK_SIZE" {
+  type        = number
+  description = "size of disk in bytes"
+}
+
+variable "OUTPUT_DIR" {
+  type        = string
+  description = "output dir"
+}
+
+variable "BOX_NAME" {
+  type        = string
+  description = "name of box being created"
+}
+
+variable "BOX_VERSION" {
+  type        = string
+  description = "version of box being created"
+}
+
+source "qemu" "cits3007" {
+
+  iso_url           = "file:///${var.SOURCE_PATH}"
+  disk_image        = true
+  format            = "qcow2"
+  iso_checksum      = "md5:${var.SOURCE_MD5}"
+
+  shutdown_command  = "sudo shutdown -P now"
+
+  communicator      = "ssh"
+  ssh_username      = "vagrant"
+  ssh_password      = "vagrant"
+  ssh_timeout       = "60m"
+
+  headless          = true
+
+  # Needn't specify an accelerator - packer docco
+  # says kvm will be used by default if available,
+  # else tcg: https://www.packer.io/docs/builders/qemu.
+
+  #accelerator       = "kvm"
+
+  output_directory  = "${var.OUTPUT_DIR}"
+
+  disk_size         = "${var.DISK_SIZE}b"
+  vm_name           = "${var.BOX_NAME}_${var.BOX_VERSION}.qcow2"
+
+  net_device        = "virtio-net"
+  disk_interface    = "virtio-scsi"
+  boot_wait         = "20s"
+
+  display           = "none"
+
+  # needed, see https://github.com/hashicorp/packer/issues/8693
+  # (??still)
+  qemuargs         = [
+      ["-display", "none"]
+    ]
+}
+
+
+build {
+  sources = ["source.qemu.cits3007"]
+
+  provisioner "shell" {
+      scripts = [
+        "../provision-01.sh",
+        "../provision-02.sh"
+      ]
+      execute_command = "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+      timeout = "60m"
+      max_retries = 2
+  }
+
+  post-processors {
+
+    post-processor "vagrant" {
+
+       compression_level = 9
+       keep_input_artifact = true
+       vagrantfile_template = "developer.rb"
+       output = "${var.OUTPUT_DIR}/${var.BOX_NAME}_${var.BOX_VERSION}.box"
+       include = [
+           "info.json"
+       ]
+    }
+
+    post-processor "checksum" {
+      checksum_types = ["md5"]
+      output = "${var.OUTPUT_DIR}/${var.BOX_NAME}_${var.BOX_VERSION}.box.md5"
+    }
+
+  }
+
+}

provision-01.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# should be run as root
+
+set -x
+
+apt-get update
+
+# should already have: wget, curl
+
+# basic apps
+
+DEBIAN_FRONTEND=noninteractive \
+  apt-get install --no-install-recommends -y \
+    apt-transport-https     \
+    aptitude                \
+    bash                    \
+    bash-completion         \
+    bzip2                   \
+    ca-certificates         \
+    command-not-found       \
+    expect                  \
+    file                    \
+    fakeroot                \
+    gpg                     \
+    jq                      \
+    less                    \
+    lsof                    \
+    lynx                    \
+    netcat-openbsd          \
+    procps                  \
+    pv                      \
+    openssh-client          \
+    screen                  \
+    sudo                    \
+    time
+
+# extra utils
+
+DEBIAN_FRONTEND=noninteractive \
+  apt-get install --no-install-recommends -y \
+    binutils                \
+    bsdmainutils            \
+    coreutils               \
+    diffutils               \
+    findutils               \
+    moreutils               \
+    patchutils              \
+    sharutils
+

provision-02.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# should be run as root
+
+set -x
+
+apt-get update
+
+# should already have: wget, curl
+
+
+# dev tools
+
+DEBIAN_FRONTEND=noninteractive \
+  apt-get install --no-install-recommends -y \
+    afl++-clang             \
+    build-essential         \
+    clang                   \
+    clang-format            \
+    clang-tidy              \
+    clang-tools             \
+    g++-multilib            \
+    gdb                     \
+    git                     \
+    gpg                     \
+    indent                  \
+    libtool                 \
+    llvm-10-dev             \
+    pkg-config              \
+    splint                  \
+    universal-ctags         \
+    valgrind                \
+    xxd                     \
+    zzuf
+
+# docco
+
+DEBIAN_FRONTEND=noninteractive \
+  apt-get install --no-install-recommends -y \
+    man-db                  \
+    manpages                \
+    manpages-dev            \
+    manpages-posix          \
+    manpages-posix-dev
+

virtualbox/publish.sh → publish.sh

@@ -12,13 +12,14 @@
 #   $ sudo add-apt-repository ppa:savoury1/curl34
 #   $ sudo apt-get install curl
 
-if (( $# != 2 )); then
-  echo >&2 "expected 2 args: PATH_TO_BOX PATH_TO_MD5"
+if (( $# != 3 )); then
+  echo >&2 "expected 3 args: PATH_TO_BOX  PATH_TO_MD5 PROVIDER"
   exit 1
 fi
 
 BOX_PATH="$1"; shift
 BOX_MD5="$(cat $1 | awk '{print $1; }')"; shift
+PROVIDER_TYPE="$1"; shift
 
 setup_colors() {
   if [[ -t 2 ]] && [[ -z "${NO_COLOR-}" ]] && [[ "${TERM-}" != "dumb" ]]; then
@@ -48,7 +49,6 @@ fi
 set -uo pipefail
 
 BOX_NAME="$(make $MAKE_ARGS print_box_name)"
-PROVIDER_TYPE="virtualbox"
 
 SHORT_DESC="$(make $MAKE_ARGS print_short_desc)"
 VAGRANT_CLOUD_USERNAME="$(make $MAKE_ARGS  print_vagrant_cloud_username)"
@@ -127,7 +127,7 @@ printf '\n'"${GREEN}%s${NOFORMAT}"'\n' "ensuring vagrant-cloud box named $VAGRAN
 cat > .provider_metadata <<END
 {
   "provider": {
-    "name": "virtualbox",
+    "name": "${PROVIDER_TYPE}",
     "checksum": "${BOX_MD5}",
     "checksum_type": "md5"
   }