update scripts and add README

Makefile

@@ -3,14 +3,9 @@ LIBRARY_PATH = $(DEPS_DIR)/usr/lib/x86_64-linux-gnu:$(DEPS_DIR)/lib/x86_64-linux
 LIBGUESTFS_PATH = guestfs/appliance
 LIBGUESTFS_HV = scripts/qemu.sh
 APPLIANCE_VERSION = 1.46.0
-DOWNLOAD_DEPS_ARGS =
 
 # Export LIBGUESTFS_DEBUG=1 to debug
 
-ifeq ($(GUESTFISH),1)
-	DOWNLOAD_DEPS_ARGS = guestfish
-endif
-
 all:
 	LIBRARY_PATH=$(LIBRARY_PATH) CGO_LDFLAGS='-l:libvirt.so.0 -l:libyajl.so.2' go build
 
@@ -18,23 +13,30 @@ run:
 	LD_LIBRARY_PATH=$(LIBRARY_PATH) LIBGUESTFS_PATH=$(LIBGUESTFS_PATH) LIBGUESTFS_HV=$(LIBGUESTFS_HV) LIBGUESTFS_BACKEND_SETTINGS=force_tcg ./cloudbuild
 
 guestfish:
-	LD_LIBRARY_PATH=$(LIBRARY_PATH) LIBGUESTFS_PATH=$(LIBGUESTFS_PATH) LIBGUESTFS_HV=$(LIBGUESTFS_HV) LIBGUESTFS_BACKEND_SETTINGS=force_tcg PATH=guestfs/qemu-utils-deps/usr/bin:$(PATH) $(DEPS_DIR)/usr/bin/guestfish
+	LD_LIBRARY_PATH=$(LIBRARY_PATH) LIBGUESTFS_PATH=$(LIBGUESTFS_PATH) LIBGUESTFS_HV=$(LIBGUESTFS_HV) LIBGUESTFS_BACKEND_SETTINGS=force_tcg scripts/guestfish.sh
 
 deps:
-	scripts/download-deps.sh $(DOWNLOAD_DEPS_ARGS)
+	scripts/download-deps.sh
+
+guestfish-deps:
+	scripts/download-guestfish-deps.sh
 
 appliance-download:
+	mkdir -p guestfs
 	cd guestfs && \
+		rm -rf appliance && \
 		wget https://download.libguestfs.org/binaries/appliance/appliance-$(APPLIANCE_VERSION).tar.xz && \
 		tar Jxvf appliance-$(APPLIANCE_VERSION).tar.xz && \
 		rm appliance-$(APPLIANCE_VERSION).tar.xz
 
-appliance: $(DEPS_DIR)usr/bin/supermin
+appliance:
-	mkdir -p /var/tmp/.guestfs-`id -u`
+	scripts/create-appliance.sh
-	$(DEPS_DIR)usr/bin/supermin --build --verbose --if-newer --lock /var/tmp/.guestfs-`id -u`/lock --copy-kernel -f ext2 --host-cpu x86_64 $(DEPS_DIR)usr/lib/x86_64-linux-gnu/guestfs/supermin.d -o /var/tmp/.guestfs-`id -u`/appliance.d
-	mv /var/tmp/.guestfs-`id -u`/appliance.d guestfs/appliance
 
-$(DEPS_DIR)usr/bin/supermin:
+supermin-download:
-	cd guestfs && apt download supermin && dpkg -x supermin_*.deb deps && rm supermin_*.deb
+	mkdir -p guestfs
+	[ -e /usr/bin/supermin ] || cd guestfs && apt download supermin && dpkg -x supermin_*.deb deps && rm supermin_*.deb
 
-.PHONY: all run guestfish deps appliance-download appliance
+clean:
+	rm -rf cloudbuild guestfs
+
+.PHONY: all run guestfish deps guestfish-deps appliance-download appliance supermin-download clean

README.md

@@ -0,0 +1,74 @@
+# cloudbuild
+cloudbuild downloads, modifies and uploads VM templates for the CSC cloud
+(CloudStack) using publicly available images for common Linux distros.
+
+## Installing dependencies
+First run
+```sh
+make deps
+```
+
+### Optional dependencies
+If you are developing cloudbuild, you may wish to install
+[guestfish](https://libguestfs.org/guestfish.1.html), an interactive shell
+for guestfs.
+
+To download the dependencies:
+```sh
+make guestfish-deps
+```
+
+To run:
+```sh
+make guestfish
+
+><fs> add /path/to/your/template.img
+><fs> run
+><fs> inspect-os
+><fs> # use the output from inspect-os to find the root partition
+><fs> mount /dev/sda4 /
+```
+
+## Creating the appliance
+Next you need an appliance, which is a kernel + initrd + rootfs used by the
+VM created by guestfs. You have two options:
+
+1. Download an appliance:
+   ```sh
+   make appliance-download
+   ```
+**OR**
+
+2. Create an appliance using the host's kernel:
+   ```sh
+   make supermin-download
+   make appliance
+   ```
+
+## Building the program
+```sh
+make
+```
+
+## Running the program
+Make sure the following environment variables are set:
+```sh
+# these values must be obtained from the CloudStack web UI
+export CLOUDSTACK_API_KEY=secret
+export CLOUDSTACK_SECRET_KEY=secret
+# set these to the distros for which cloudbuild will create templates
+export DISTROS_TO_CHECK=ubuntu,fedora
+# the modified templates will be temporarily moved here so that CloudStack
+# can download them
+export UPLOAD_DIRECTORY=/var/www/csc-cloud-images
+# the modified templates (in the UPLOAD_DIRECTORY) must be downloadable
+# from this base URL
+export UPLOAD_BASE_URL=http://biloba.csclub.uwaterloo.ca/csc-cloud-images
+# notification emails will be sent here
+export EMAIL_RECIPIENT=root@csclub.uwaterloo.ca
+```
+
+Finally:
+```sh
+make run
+```

scripts/create-appliance.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+DEPS_DIR=guestfs/deps
+SUPERMIN=/usr/bin/supermin
+INPUT_DIR=/usr/lib/x86_64-linux-gnu/guestfs/supermin.d
+if ! [ -e $SUPERMIN ]; then
+    SUPERMIN=${DEPS_DIR}${SUPERMIN}
+fi
+if ! [ -e $INPUT_DIR ]; then
+    INPUT_DIR=${DEPS_DIR}${INPUT_DIR}
+fi
+
+set -x
+mkdir -p /var/tmp/.guestfs-`id -u`
+$SUPERMIN --build --verbose --if-newer \
+    --lock /var/tmp/.guestfs-`id -u`/lock \
+    --copy-kernel \
+    -f ext2 \
+    --host-cpu x86_64 \
+    $INPUT_DIR \
+    -o /var/tmp/.guestfs-`id -u`/appliance.d
+mkdir -p guestfs
+rm -rf guestfs/appliance
+mv /var/tmp/.guestfs-`id -u`/appliance.d guestfs/appliance

scripts/download-deps.sh

@@ -2,19 +2,22 @@
 
 set -e
 
+core_dependencies=(
+    golang-guestfs-dev
+    libguestfs0
+    libguestfs-dev
+)
+
 # This worked on Debian bullseye on the CSC machines - will need to
 # be updated for other versions or other distros
-# TODO: automatically generate this list from the core dependencies
+# TODO: automatically generate this list from the dependencies for
-# (golang-guestfs-dev, libvirt0, qemu-system-x86)
+# golang-guestfs-dev, libvirt0, qemu-system-x86
-dependencies=(
+libvirt_dependencies=(
-    golang-guestfs-dev
+    libbrlapi0.8
-    libbrlapi0
     libcacard0
     libcapstone4
     libdaxctl1
     libfdt1
-    libguestfs0
-    libguestfs-dev
     libndctl6
     libpmem1
     libslirp0
@@ -29,19 +32,16 @@ dependencies=(
     libxenevtchn1
     libxenforeignmemory1
     libxengnttab1
-    libxenmisc4
+    libxenmisc4.14
-    libxenstore3
+    libxenstore3.0
     libxentoolcore1
     libxentoollog1
     libyajl2
     qemu-system-x86
-    qemu-utils
     seabios
 )
 
-if [ $# -eq 1 ] && [ "$1" = guestfish ]; then
+dependencies=("${core_dependencies[@]}" "${libvirt_dependencies[@]}")
-    dependencies=("${dependencies[@]}" libguestfs-tools libconfig9)
-fi
 
 is_installed() {
     status=$(dpkg-query --show --showformat '${db:Status-Status}' "$1" 2>/dev/null)
@@ -51,16 +51,18 @@ is_installed() {
 DEPS_DIR=guestfs/deps
 mkdir -p $DEPS_DIR
 cd guestfs
-
 for dep in "${dependencies[@]}"; do
+    if is_installed $dep; then
+        continue
+    fi
     set -x
     apt download $dep
     dpkg -x ${dep}_*.deb deps
     set +x
 done
 set -x
-rm *.deb
+rm -f *.deb
 
-cp $DEPS_DIR/usr/share/gocode/src/libguestfs.org/guestfs/guestfs.go .
+cp deps/usr/share/gocode/src/libguestfs.org/guestfs/guestfs.go .
-cp $DEPS_DIR/usr/include/guestfs.h .
+cp deps/usr/include/guestfs.h .
 go mod init guestfs

scripts/download-guestfish-deps.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+dependencies=(
+    libguestfs-tools
+    libconfig9
+    qemu-utils
+)
+
+is_installed() {
+    status=$(dpkg-query --show --showformat '${db:Status-Status}' "$1" 2>/dev/null)
+    [ $? -eq 0 ] && [ "$status" = installed ]
+}
+
+mkdir -p guestfs/deps
+cd guestfs
+for dep in "${dependencies[@]}"; do
+    if is_installed $dep; then
+        continue
+    fi
+    set -x
+    apt download $dep
+    dpkg -x ${dep}_*.deb deps
+    set +x
+done
+set -x
+rm -f *.deb

scripts/guestfish.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+DEPS_DIR=guestfs/deps
+GUESTFISH=/usr/bin/guestfish
+if ! [ -e $GUESTFISH ]; then
+    GUESTFISH=${DEPS_DIR}${GUESTFISH}
+fi
+if ! [ -e /usr/bin/qemu-img ]; then
+    export PATH="$DEPS_DIR/usr/bin:$PATH"
+fi
+set -x
+exec $GUESTFISH

scripts/qemu.sh

@@ -1,7 +1,8 @@
 #!/bin/sh
 
-if [ -x /usr/bin/qemu-system-x86_64 ]; then
+QEMU=/usr/bin/qemu-system-x86_64
-    exec /usr/bin/qemu-system-x86_64 "$@"
+if [ -x $QEMU ]; then
+    exec $QEMU "$@"
 fi
 DEPS_DIR=guestfs/deps
-exec $DEPS_DIR/usr/bin/qemu-system-x86_64 -L $DEPS_DIR/usr/share/seabios "$@"
+exec ${DEPS_DIR}${QEMU} -L $DEPS_DIR/usr/share/seabios "$@"