first commit
This commit is contained in:
commit
08f7371198
|
|
@ -0,0 +1,3 @@
|
|||
/cloudbuild
|
||||
/guestfs
|
||||
*.swp
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
DEPS_DIR = guestfs/deps
|
||||
LIBRARY_PATH = $(DEPS_DIR)/usr/lib/x86_64-linux-gnu:$(DEPS_DIR)/lib/x86_64-linux-gnu
|
||||
LIBGUESTFS_PATH = guestfs/appliance
|
||||
LIBGUESTFS_HV = scripts/qemu.sh
|
||||
APPLIANCE_VERSION = 1.46.0
|
||||
DOWNLOAD_DEPS_ARGS =
|
||||
|
||||
# Export LIBGUESTFS_DEBUG=1 to debug
|
||||
|
||||
ifeq ($(GUESTFISH),1)
|
||||
DOWNLOAD_DEPS_ARGS = guestfish
|
||||
endif
|
||||
|
||||
all:
|
||||
LIBRARY_PATH=$(LIBRARY_PATH) CGO_LDFLAGS='-l:libvirt.so.0 -l:libyajl.so.2' go build
|
||||
|
||||
run:
|
||||
LD_LIBRARY_PATH=$(LIBRARY_PATH) LIBGUESTFS_PATH=$(LIBGUESTFS_PATH) LIBGUESTFS_HV=$(LIBGUESTFS_HV) LIBGUESTFS_BACKEND_SETTINGS=force_tcg ./cloudbuild
|
||||
|
||||
guestfish:
|
||||
LD_LIBRARY_PATH=$(LIBRARY_PATH) LIBGUESTFS_PATH=$(LIBGUESTFS_PATH) LIBGUESTFS_HV=$(LIBGUESTFS_HV) LIBGUESTFS_BACKEND_SETTINGS=force_tcg PATH=guestfs/qemu-utils-deps/usr/bin:$(PATH) $(DEPS_DIR)/usr/bin/guestfish
|
||||
|
||||
deps:
|
||||
scripts/download-deps.sh $(DOWNLOAD_DEPS_ARGS)
|
||||
|
||||
appliance-download:
|
||||
cd guestfs && \
|
||||
wget https://download.libguestfs.org/binaries/appliance/appliance-$(APPLIANCE_VERSION).tar.xz && \
|
||||
tar Jxvf appliance-$(APPLIANCE_VERSION).tar.xz && \
|
||||
rm appliance-$(APPLIANCE_VERSION).tar.xz
|
||||
|
||||
appliance: $(DEPS_DIR)usr/bin/supermin
|
||||
mkdir -p /var/tmp/.guestfs-`id -u`
|
||||
$(DEPS_DIR)usr/bin/supermin --build --verbose --if-newer --lock /var/tmp/.guestfs-`id -u`/lock --copy-kernel -f ext2 --host-cpu x86_64 $(DEPS_DIR)usr/lib/x86_64-linux-gnu/guestfs/supermin.d -o /var/tmp/.guestfs-`id -u`/appliance.d
|
||||
mv /var/tmp/.guestfs-`id -u`/appliance.d guestfs/appliance
|
||||
|
||||
$(DEPS_DIR)usr/bin/supermin:
|
||||
cd guestfs && apt download supermin && dpkg -x supermin_*.deb deps && rm supermin_*.deb
|
||||
|
||||
.PHONY: all run guestfish deps appliance-download appliance
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
module git.csclub.uwaterloo.ca/cloud/cloudbuild
|
||||
|
||||
go 1.17
|
||||
|
||||
replace libguestfs.org/guestfs => ./guestfs
|
||||
|
||||
require (
|
||||
github.com/rs/zerolog v1.26.1
|
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d
|
||||
libguestfs.org/guestfs v0.0.0-00010101000000-000000000000
|
||||
)
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc=
|
||||
github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc=
|
||||
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
|
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/cloudbuilder"
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config"
|
||||
)
|
||||
|
||||
func setupLogging() {
|
||||
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339})
|
||||
}
|
||||
|
||||
func main() {
|
||||
setupLogging()
|
||||
cfg := config.New()
|
||||
builder := cloudbuilder.New(cfg)
|
||||
|
||||
if err := builder.Start(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,287 @@
|
|||
package cloudbuilder
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
_ "embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/smtp"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/cloudstack"
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config"
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/distros"
|
||||
)
|
||||
|
||||
//go:embed email-message.txt
|
||||
var emailMessageTemplate string
|
||||
|
||||
type CloudBuilder struct {
|
||||
cfg *config.Config
|
||||
client *cloudstack.CloudstackClient
|
||||
}
|
||||
|
||||
func New(cfg *config.Config) *CloudBuilder {
|
||||
return &CloudBuilder{
|
||||
cfg: cfg,
|
||||
client: cloudstack.New(cfg),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *CloudBuilder) moveTemplateToUploadArea(templatePath string) (newPath string, err error) {
|
||||
newPath = path.Join(c.cfg.UploadDirectory, path.Base(templatePath))
|
||||
log.Debug().Msg(fmt.Sprintf("Moving %s to %s", templatePath, newPath))
|
||||
err = os.Rename(templatePath, newPath)
|
||||
return
|
||||
}
|
||||
|
||||
func (c *CloudBuilder) UploadTemplate(
|
||||
templatePath string, templateName string, osDescription string,
|
||||
) (template *cloudstack.Template, err error) {
|
||||
templatePath, err = c.moveTemplateToUploadArea(templatePath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
osTypeId, err := c.client.GetOsTypeId(osDescription)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
uploadUrl := strings.TrimRight(c.cfg.UploadBaseUrl, "/") + "/" + path.Base(templatePath)
|
||||
log.Debug().Msg(fmt.Sprintf("Uploading new template from %s", uploadUrl))
|
||||
template, err = c.client.RegisterTemplate(templateName, uploadUrl, osTypeId)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
const pollInterval = 5 * time.Second
|
||||
// Due to clock drift, we may end up waiting a bit longer than this
|
||||
const maxTimeToWait = 5 * time.Minute
|
||||
var timeWaited time.Duration = 0
|
||||
for !template.IsReady {
|
||||
// TODO: make sure that DownloadDetails[0] exists
|
||||
log.Debug().
|
||||
Str("templateName", template.Name).
|
||||
Str("status", template.Status).
|
||||
Str("downloadPercent", template.DownloadDetails[0].DownloadPercent).
|
||||
Str("downloadState", template.DownloadDetails[0].DownloadState).
|
||||
Msg("Template is not ready yet...")
|
||||
time.Sleep(pollInterval)
|
||||
timeWaited += pollInterval
|
||||
if timeWaited >= maxTimeToWait {
|
||||
err = errors.New("Timed out waiting for template to be ready")
|
||||
return
|
||||
}
|
||||
template, err = c.client.GetTemplate(template.Id)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
log.Debug().Msg("Deleting " + templatePath)
|
||||
err = os.Remove(templatePath)
|
||||
return
|
||||
}
|
||||
|
||||
func (c *CloudBuilder) CreateVM(
|
||||
name string, templateID string,
|
||||
) (virtualMachine *cloudstack.VirtualMachine, err error) {
|
||||
vmID, err := c.client.DeployVirtualMachine(name, templateID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
virtualMachine, err = c.client.GetVirtualMachine(vmID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
const pollInterval = 10 * time.Second
|
||||
// Due to clock drift, we may end up waiting a bit longer than this
|
||||
const maxTimeToWait = 30 * time.Minute
|
||||
var timeWaited time.Duration = 0
|
||||
log.Debug().Str("templateID", templateID).Msg("Creating new VM")
|
||||
for virtualMachine.State != "Running" {
|
||||
log.Debug().
|
||||
Str("name", name).
|
||||
Str("id", virtualMachine.Id).
|
||||
Str("state", virtualMachine.State).
|
||||
Msg("VM is not ready yet...")
|
||||
time.Sleep(pollInterval)
|
||||
timeWaited += pollInterval
|
||||
if timeWaited >= maxTimeToWait {
|
||||
err = errors.New("Timed out waiting for VM to be ready")
|
||||
return
|
||||
}
|
||||
virtualMachine, err = c.client.GetVirtualMachine(vmID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
log.Debug().Msg("VM successfully created")
|
||||
return
|
||||
}
|
||||
|
||||
type DistroInfo struct {
|
||||
name string
|
||||
manager distros.ITemplateManager
|
||||
osDescription string
|
||||
user string
|
||||
}
|
||||
|
||||
func (c *CloudBuilder) getExistingTemplateVersions() map[string]string {
|
||||
templatePattern := regexp.MustCompile("^CSC (?P<distro>[A-Za-z][A-Za-z ]+[A-Za-z]) (?P<version>\\d+(\\.\\d+)?)$")
|
||||
mostRecentVersions := make(map[string]string)
|
||||
templates := c.client.ListTemplates()
|
||||
for _, template := range templates {
|
||||
submatches := templatePattern.FindStringSubmatch(template.Name)
|
||||
if submatches == nil {
|
||||
log.Debug().Msg(fmt.Sprintf("Template '%s' did not match pattern, skipping", template.Name))
|
||||
continue
|
||||
}
|
||||
distro := submatches[1]
|
||||
version := submatches[2]
|
||||
log.Debug().Str("distro", distro).Str("version", version).Msg("Found template")
|
||||
otherVersion, ok := mostRecentVersions[distro]
|
||||
if !ok || version > otherVersion {
|
||||
mostRecentVersions[distro] = version
|
||||
}
|
||||
}
|
||||
return mostRecentVersions
|
||||
}
|
||||
|
||||
func (c *CloudBuilder) sendEmailNotification(
|
||||
templateName string, vm *cloudstack.VirtualMachine, vmUser string,
|
||||
) (err error) {
|
||||
tmpl := template.Must(template.New("email-message").Parse(emailMessageTemplate))
|
||||
data := map[string]interface{}{
|
||||
"cfg": c.cfg,
|
||||
"date": time.Now().Format(time.RFC1123Z),
|
||||
"templateName": templateName,
|
||||
"vm": vm,
|
||||
"vmUser": vmUser,
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err = tmpl.Execute(&buf, data); err != nil {
|
||||
return
|
||||
}
|
||||
// The lines of the body need to be CRLF terminated
|
||||
// See https://pkg.go.dev/net/smtp#SendMail
|
||||
// Make sure that email-message.txt uses Unix-style LF endings
|
||||
msg := bytes.ReplaceAll(buf.Bytes(), []byte("\n"), []byte("\r\n"))
|
||||
log.Debug().
|
||||
Str("to", c.cfg.EmailRecipient).
|
||||
Msg("sending email notification")
|
||||
return smtp.SendMail(
|
||||
c.cfg.EmailServer,
|
||||
nil, /* auth */
|
||||
c.cfg.EmailSender,
|
||||
[]string{c.cfg.EmailRecipient},
|
||||
msg,
|
||||
)
|
||||
}
|
||||
|
||||
func (c *CloudBuilder) createNewTemplate(
|
||||
newVersion, codename string, distroInfo *DistroInfo,
|
||||
) (template *cloudstack.Template, err error) {
|
||||
distroManager := distroInfo.manager
|
||||
templatePath, err := distroManager.DownloadTemplate(codename)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err = distroManager.ModifyTemplate(templatePath); err != nil {
|
||||
return
|
||||
}
|
||||
templateName := fmt.Sprintf("CSC %s %s", distroInfo.name, newVersion)
|
||||
template, err = c.UploadTemplate(templatePath, templateName, distroInfo.osDescription)
|
||||
return
|
||||
}
|
||||
|
||||
func (c *CloudBuilder) versionStringCompare(version1, version2 string) int {
|
||||
f1, err := strconv.ParseFloat(version1, 32)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
f2, err := strconv.ParseFloat(version2, 32)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if f1 < f2 {
|
||||
return -1
|
||||
} else if f1 > f2 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (c *CloudBuilder) Start() (err error) {
|
||||
distrosInfo := map[string]DistroInfo{
|
||||
"Ubuntu": DistroInfo{
|
||||
name: "Ubuntu",
|
||||
manager: distros.NewUbuntuTemplateManager(c.cfg),
|
||||
osDescription: "Other Ubuntu (64-bit)",
|
||||
user: "ubuntu",
|
||||
},
|
||||
"Fedora": DistroInfo{
|
||||
name: "Fedora",
|
||||
manager: distros.NewFedoraTemplateManager(c.cfg),
|
||||
osDescription: "Fedora Linux (64 bit)",
|
||||
user: "fedora",
|
||||
},
|
||||
"AlmaLinux": DistroInfo{
|
||||
name: "AlmaLinux",
|
||||
manager: distros.NewAlmaLinuxTemplateManager(c.cfg),
|
||||
osDescription: "Other CentOS (64-bit)",
|
||||
user: "almalinux",
|
||||
},
|
||||
}
|
||||
// The elements of this slice must be keys of the map above
|
||||
// TODO use environment variable
|
||||
distrosToCheck := []string{"AlmaLinux"}
|
||||
existingVersions := c.getExistingTemplateVersions()
|
||||
for _, distro := range distrosToCheck {
|
||||
distroInfo := distrosInfo[distro]
|
||||
distroManager := distroInfo.manager
|
||||
var newVersion, codename string
|
||||
newVersion, codename, err = distroManager.GetLatestVersion()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
log.Debug().Str("newVersion", newVersion).Str("codename", codename).Msg(distro)
|
||||
|
||||
curVersion, ok := existingVersions[distro]
|
||||
if ok && c.versionStringCompare(newVersion, curVersion) <= 0 {
|
||||
log.Debug().
|
||||
Str("distro", distro).
|
||||
Str("version", curVersion).
|
||||
Msg("Existing version is up to date, skipping")
|
||||
continue
|
||||
}
|
||||
log.Info().Str("distro", distro).Msg("Existing template is out of date, creating a new one")
|
||||
|
||||
var template *cloudstack.Template
|
||||
template, err = c.createNewTemplate(newVersion, codename, &distroInfo)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
vmName := strings.Join([]string{
|
||||
strings.ReplaceAll(strings.ToLower(distro), " ", "-"),
|
||||
strings.ReplaceAll(newVersion, ".", "-"),
|
||||
"test",
|
||||
}, "-")
|
||||
var vm *cloudstack.VirtualMachine
|
||||
vm, err = c.CreateVM(vmName, template.Id)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err = c.sendEmailNotification(template.Name, vm, distroInfo.user); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
From: {{ .cfg.EmailSenderName }} <{{ .cfg.EmailSender }}>
|
||||
To: {{ .cfg.EmailRecipient }}
|
||||
Reply-To: {{ .cfg.EmailReplyTo }}
|
||||
Subject: New VM Template: {{ .templateName }}
|
||||
Date: {{ .date }}
|
||||
|
||||
Hello syscom,
|
||||
|
||||
This is an automated message from cloudbuild, the CSC VM template
|
||||
builder.
|
||||
|
||||
A new VM template, {{ .templateName }}, has been uploaded to CloudStack.
|
||||
It is not public. A new VM, {{ .vm.Name }}, has been created from this
|
||||
template. You can SSH into this VM from biloba or chamomile by running
|
||||
the following:
|
||||
|
||||
ssh -i /var/lib/cloudstack/management/.ssh/id_rsa {{ .vmUser }}@{{ (index .vm.Nic 0).IpAddress }}
|
||||
|
||||
Please login to the VM and verify that everything is working correctly.
|
||||
Once you have done this, please login to CloudStack with the admin account
|
||||
and perform the following:
|
||||
|
||||
1. Delete the VM (enable the "Expunge" option too).
|
||||
2. Make the template public:
|
||||
From the web page for the template, click the "Update Template Sharing"
|
||||
circular button in the top right corner, and toggle the "Public"
|
||||
and "Featured" sliders.
|
||||
3. Delete the old template (if it is not being used by any VMs):
|
||||
From the web page for the old template, click the "Zones" tab, then
|
||||
press the red circular "Delete" button beside "{{ .cfg.CloudstackZoneName }}".
|
||||
|
||||
Sincerely,
|
||||
cloudbuild
|
||||
|
|
@ -0,0 +1,399 @@
|
|||
// Package cloudstack provides utilities for interacting with a CloudStack
|
||||
// management server.
|
||||
package cloudstack
|
||||
|
||||
/*
|
||||
* See https://cloudstack.apache.org/api/apidocs-4.16/ for API docs
|
||||
*/
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config"
|
||||
)
|
||||
|
||||
// A CloudstackClient interacts with a CloudStack management server via the
|
||||
// REST API.
|
||||
type CloudstackClient struct {
|
||||
cfg *config.Config
|
||||
cachedZoneId string
|
||||
cachedServiceOfferingId string
|
||||
}
|
||||
|
||||
// New returns a CloudStack client with the given config values.
|
||||
func New(cfg *config.Config) *CloudstackClient {
|
||||
return &CloudstackClient{cfg: cfg}
|
||||
}
|
||||
|
||||
// CloudStack uses java.net.URLEncoder when calculating signatures:
|
||||
// https://github.com/apache/cloudstack/blob/main/server/src/main/java/com/cloud/api/ApiServer.java
|
||||
// Unfortunately java.net.URLEncoder transforms "~" into "%7E".
|
||||
// This is a bug: https://bugs.openjdk.org/browse/JDK-8204530
|
||||
// So, we need to match Java's behaviour so that the server does not
|
||||
// reject our signature.
|
||||
// (I noticed this problem when passing a `url` parameter to registerTemplate
|
||||
// which contained a tilde.)
|
||||
// TODO: file a GitHub issue in the cloudmonkey repo
|
||||
func urlQueryEscape(value string) string {
|
||||
return strings.ReplaceAll(url.QueryEscape(value), "~", "%7E")
|
||||
}
|
||||
|
||||
// Adapted from https://github.com/apache/cloudstack-cloudmonkey/blob/main/cmd/network.go
|
||||
func encodeRequestParams(params map[string]string) string {
|
||||
if params == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(params))
|
||||
for key := range params {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var buf bytes.Buffer
|
||||
for _, key := range keys {
|
||||
value := params[key]
|
||||
if buf.Len() > 0 {
|
||||
buf.WriteByte('&')
|
||||
}
|
||||
buf.WriteString(key)
|
||||
buf.WriteString("=")
|
||||
buf.WriteString(urlQueryEscape(value))
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// See http://docs.cloudstack.apache.org/en/4.16.0.0/developersguide/dev.html#how-to-sign-an-api-call-with-python
|
||||
func (client *CloudstackClient) createURL(params map[string]string) string {
|
||||
cfg := client.cfg
|
||||
params["apiKey"] = cfg.CloudstackApiKey
|
||||
params["response"] = "json"
|
||||
|
||||
// adapted from https://github.com/apache/cloudstack-cloudmonkey/blob/main/cmd/network.go
|
||||
encodedParams := encodeRequestParams(params)
|
||||
mac := hmac.New(sha1.New, []byte(cfg.CloudstackSecretKey))
|
||||
mac.Write([]byte(strings.Replace(strings.ToLower(encodedParams), "+", "%20", -1)))
|
||||
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil))
|
||||
encodedParams += fmt.Sprintf("&signature=%s", urlQueryEscape(signature))
|
||||
|
||||
return fmt.Sprintf("%s?%s", cfg.CloudstackApiBaseUrl, encodedParams)
|
||||
}
|
||||
|
||||
func getDeserializedResponse(url string, unmarshaledValue interface{}) {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
buf, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err = json.Unmarshal(buf, unmarshaledValue); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
type ErrorInfo struct {
|
||||
ErrorCode int `json:"errorcode"`
|
||||
ErrorText string `json:"errortext"`
|
||||
}
|
||||
|
||||
func checkErrorInfo(data *ErrorInfo) {
|
||||
if data.ErrorCode != 0 {
|
||||
panic(fmt.Sprintf("Error %d: %s", data.ErrorCode, data.ErrorText))
|
||||
}
|
||||
}
|
||||
|
||||
type ListDomainsResponse struct {
|
||||
ErrorInfo
|
||||
Count int `json:"count"`
|
||||
Domain []struct {
|
||||
Name string `json:"name"`
|
||||
Id string `json:"id"`
|
||||
} `json:"domain"`
|
||||
}
|
||||
|
||||
// GetDomainId returns the ID of the domain with the given name.
|
||||
func (client *CloudstackClient) GetDomainId(domainName string) string {
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "listDomains",
|
||||
"details": "min",
|
||||
"name": domainName,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response ListDomainsResponse `json:"listdomainsresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
if data.Count != 1 {
|
||||
panic("there should be one domain found")
|
||||
}
|
||||
return data.Domain[0].Id
|
||||
}
|
||||
|
||||
type Template struct {
|
||||
Created string `json:"created"`
|
||||
Domain string `json:"domain"`
|
||||
DomainId string `json:"domainid"`
|
||||
Id string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
IsReady bool `json:"isready"`
|
||||
IsFeatured bool `json:"isfeatured"`
|
||||
IsPublic bool `json:"ispublic"`
|
||||
Status string `json:"status"`
|
||||
DownloadDetails []struct {
|
||||
DownloadPercent string `json:"downloadPercent"`
|
||||
DownloadState string `json:"downloadState"`
|
||||
} `json:"downloaddetails"`
|
||||
}
|
||||
|
||||
type ListTemplatesResponse struct {
|
||||
ErrorInfo
|
||||
Count int `json:"count"`
|
||||
Template []Template `json:"template"`
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) listTemplates(templateID string) []Template {
|
||||
params := map[string]string{
|
||||
"command": "listTemplates",
|
||||
"details": "min",
|
||||
"templatefilter": "self",
|
||||
}
|
||||
if templateID != "" {
|
||||
params["id"] = templateID
|
||||
}
|
||||
url := client.createURL(params)
|
||||
responseWrapper := struct {
|
||||
Response ListTemplatesResponse `json:"listtemplatesresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
return data.Template
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) ListTemplates() []Template {
|
||||
return client.listTemplates("")
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) GetTemplate(templateID string) (*Template, error) {
|
||||
templates := client.listTemplates(templateID)
|
||||
if len(templates) == 0 {
|
||||
return nil, errors.New("Template not found")
|
||||
}
|
||||
return &templates[0], nil
|
||||
}
|
||||
|
||||
type ListZonesResponse struct {
|
||||
ErrorInfo
|
||||
Count int `json:"count"`
|
||||
Zone []struct {
|
||||
Id string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"zone"`
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) getZoneId() (string, error) {
|
||||
if client.cachedZoneId != "" {
|
||||
return client.cachedZoneId, nil
|
||||
}
|
||||
zoneName := client.cfg.CloudstackZoneName
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "listZones",
|
||||
"name": zoneName,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response ListZonesResponse `json:"listzonesresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
if data.Count != 1 {
|
||||
return "", errors.New(fmt.Sprintf("Expected 1 zone for '%s'; got %d", zoneName, data.Count))
|
||||
}
|
||||
zoneId := data.Zone[0].Id
|
||||
client.cachedZoneId = zoneId
|
||||
return zoneId, nil
|
||||
}
|
||||
|
||||
type ListServiceOfferingsResponse struct {
|
||||
ErrorInfo
|
||||
Count int `json:"count"`
|
||||
ServiceOffering []struct {
|
||||
Id string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"serviceoffering"`
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) getServiceOfferingId() (string, error) {
|
||||
if client.cachedServiceOfferingId != "" {
|
||||
return client.cachedServiceOfferingId, nil
|
||||
}
|
||||
serviceOfferingName := client.cfg.CloudstackServiceOfferingName
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "listServiceOfferings",
|
||||
"name": serviceOfferingName,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response ListServiceOfferingsResponse `json:"listserviceofferingsresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
if data.Count != 1 {
|
||||
return "", errors.New(fmt.Sprintf("Expected 1 service offering for '%s'; got %d", serviceOfferingName, data.Count))
|
||||
}
|
||||
serviceOfferingId := data.ServiceOffering[0].Id
|
||||
client.cachedServiceOfferingId = serviceOfferingId
|
||||
return serviceOfferingId, nil
|
||||
}
|
||||
|
||||
// Note: if an OS type you need isn't available in CloudStack, you can
|
||||
// create a new one with addGuestOs
|
||||
|
||||
type OsType struct {
|
||||
Id string `json:"id"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
type ListOsTypesResponse struct {
|
||||
ErrorInfo
|
||||
Count int `json:"count"`
|
||||
OsType []OsType `json:"ostype"`
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) GetOsTypeId(osDescription string) (string, error) {
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "listOsTypes",
|
||||
"description": osDescription,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response ListOsTypesResponse `json:"listostypesresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
if data.Count != 1 {
|
||||
return "", errors.New(fmt.Sprintf("Expected 1 OS type for '%s'; got %d", osDescription, data.Count))
|
||||
}
|
||||
return data.OsType[0].Id, nil
|
||||
}
|
||||
|
||||
type RegisterTemplateResponse struct {
|
||||
ErrorInfo
|
||||
Count int `json:"count"`
|
||||
Template []Template `json:"template"`
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) RegisterTemplate(name string, downloadUrl string, osTypeId string) (*Template, error) {
|
||||
zoneId, err := client.getZoneId()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "registerTemplate",
|
||||
"displaytext": name,
|
||||
"format": "QCOW2",
|
||||
"hypervisor": "KVM",
|
||||
"name": name,
|
||||
"url": downloadUrl,
|
||||
"isextractable": "true",
|
||||
"isfeatured": "false",
|
||||
"ispublic": "false",
|
||||
"ostypeid": osTypeId,
|
||||
"requireshvm": "true",
|
||||
"zoneid": zoneId,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response RegisterTemplateResponse `json:"registertemplateresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
return &data.Template[0], nil
|
||||
}
|
||||
|
||||
type VirtualMachine struct {
|
||||
Id string `json:"id"`
|
||||
DisplayName string `json:"displayname"`
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
Nic []struct {
|
||||
Id string `json:"id"`
|
||||
IpAddress string `json:"ipaddress"`
|
||||
IsDefault bool `json:"isdefault"`
|
||||
} `json:"nic"`
|
||||
}
|
||||
|
||||
type DeployVirtualMachineResponse struct {
|
||||
ErrorInfo
|
||||
Id string `json:"id"`
|
||||
JobId string `json:"jobid"`
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) DeployVirtualMachine(
|
||||
name string, templateID string,
|
||||
) (vmID string, err error) {
|
||||
serviceOfferingId, err := client.getServiceOfferingId()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
zoneId, err := client.getZoneId()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "deployVirtualMachine",
|
||||
"serviceofferingid": serviceOfferingId,
|
||||
"templateid": templateID,
|
||||
"zoneid": zoneId,
|
||||
"name": name,
|
||||
"displayname": name,
|
||||
"keypair": client.cfg.CloudstackKeypairName,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response DeployVirtualMachineResponse `json:"deployvirtualmachineresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
return data.Id, nil
|
||||
}
|
||||
|
||||
type ListVirtualMachinesResponse struct {
|
||||
ErrorInfo
|
||||
Count int `json:"count"`
|
||||
VirtualMachine []VirtualMachine `json:"virtualmachine"`
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) GetVirtualMachine(vmID string) (virtualMachine *VirtualMachine, err error) {
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "listVirtualMachines",
|
||||
"id": vmID,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response ListVirtualMachinesResponse `json:"listvirtualmachinesresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
virtualMachine = &data.VirtualMachine[0]
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
// Package config provides utilities for reading and storing configuration
|
||||
// values.
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// A Config holds all of the configuration values needed for the program.
|
||||
type Config struct {
|
||||
CloudstackApiKey string
|
||||
CloudstackSecretKey string
|
||||
CloudstackApiBaseUrl string
|
||||
CloudstackZoneName string
|
||||
CloudstackServiceOfferingName string
|
||||
CloudstackKeypairName string
|
||||
UploadDirectory string
|
||||
UploadBaseUrl string
|
||||
MirrorHost string
|
||||
EmailServer string
|
||||
EmailSender string
|
||||
EmailSenderName string
|
||||
EmailRecipient string
|
||||
EmailReplyTo string
|
||||
}
|
||||
|
||||
// New returns a Config filled with values read from environment variables.
|
||||
// It panics if a required environment variable is empty or not set.
|
||||
func New() *Config {
|
||||
cfg := &Config{
|
||||
CloudstackApiKey: os.Getenv("CLOUDSTACK_API_KEY"),
|
||||
CloudstackSecretKey: os.Getenv("CLOUDSTACK_SECRET_KEY"),
|
||||
UploadDirectory: os.Getenv("UPLOAD_DIRECTORY"),
|
||||
UploadBaseUrl: os.Getenv("UPLOAD_BASE_URL"),
|
||||
}
|
||||
if cfg.CloudstackApiKey == "" {
|
||||
panic("CLOUDSTACK_API_KEY is empty or not set")
|
||||
}
|
||||
if cfg.CloudstackSecretKey == "" {
|
||||
panic("CLOUDSTACK_SECRET_KEY is empty or not set")
|
||||
}
|
||||
if cfg.UploadDirectory == "" {
|
||||
panic("UPLOAD_DIRECTORY is empty or not set")
|
||||
}
|
||||
if cfg.UploadBaseUrl == "" {
|
||||
panic("UPLOAD_BASE_URL is empty or not set")
|
||||
}
|
||||
|
||||
// These should never change
|
||||
cfg.CloudstackApiBaseUrl = "https://cloud.csclub.uwaterloo.ca/client/api"
|
||||
cfg.CloudstackZoneName = "Zone1"
|
||||
cfg.CloudstackServiceOfferingName = "Small Instance"
|
||||
cfg.CloudstackKeypairName = "management-keypair"
|
||||
cfg.MirrorHost = "mirror.csclub.uwaterloo.ca"
|
||||
cfg.EmailServer = "mail.csclub.uwaterloo.ca:25"
|
||||
cfg.EmailSender = "cloudbuild@csclub.uwaterloo.ca"
|
||||
cfg.EmailSenderName = "cloudbuild"
|
||||
// TODO: change recipient to syscom
|
||||
cfg.EmailRecipient = "merenber@csclub.uwaterloo.ca"
|
||||
cfg.EmailReplyTo = "no-reply@csclub.uwaterloo.ca"
|
||||
return cfg
|
||||
}
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
package distros
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"golang.org/x/net/html"
|
||||
"libguestfs.org/guestfs"
|
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config"
|
||||
)
|
||||
|
||||
// TODO: reduce code duplication with FedoraTemplateManager
|
||||
|
||||
type AlmaLinuxTemplateManager struct {
|
||||
TemplateManager
|
||||
}
|
||||
|
||||
// NewAlmaLinuxTemplateManager returns a AlmaLinuxTemplateManager with the given
|
||||
// config values.
|
||||
func NewAlmaLinuxTemplateManager(cfg *config.Config) *AlmaLinuxTemplateManager {
|
||||
logger := log.With().Str("distro", "almalinux").Logger()
|
||||
almaLinuxTemplateManager := AlmaLinuxTemplateManager{
|
||||
TemplateManager{
|
||||
cfg: cfg,
|
||||
logger: &logger,
|
||||
impl: nil,
|
||||
},
|
||||
}
|
||||
almaLinuxTemplateManager.TemplateManager.impl = &almaLinuxTemplateManager
|
||||
return &almaLinuxTemplateManager
|
||||
}
|
||||
|
||||
var floatNumberSlashPattern *regexp.Regexp = regexp.MustCompile("^\\d+(\\.\\d)?/$")
|
||||
|
||||
func getMaxFloatVersionFromHtml(node *html.Node, maxVersion *float64) {
|
||||
if isHyperlink(node) {
|
||||
text := node.FirstChild.Data
|
||||
if numberSlashPattern.FindString(text) != "" {
|
||||
version, _ := strconv.ParseFloat(text[:len(text)-1], 64)
|
||||
if version > *maxVersion {
|
||||
*maxVersion = version
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
for c := node.FirstChild; c != nil; c = c.NextSibling {
|
||||
getMaxFloatVersionFromHtml(c, maxVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) GetLatestVersion() (version string, codename string, err error) {
|
||||
resp, err := http.Get("https://mirror.csclub.uwaterloo.ca/almalinux/")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
node, err := html.Parse(resp.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var floatVersion float64
|
||||
getMaxFloatVersionFromHtml(node, &floatVersion)
|
||||
if floatVersion == 0 {
|
||||
err = errors.New("Could not determine latest AlmaLinux version from HTML")
|
||||
} else {
|
||||
version = strconv.FormatFloat(floatVersion, 'f', -1, 64)
|
||||
// AlmaLinux doesn't have codenames, only numbered versions
|
||||
codename = version
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) DownloadTemplate(codename string) (path string, err error) {
|
||||
// TODO: REMOVE THIS
|
||||
//if true {
|
||||
// return "AlmaLinux-9-GenericCloud-latest.x86_64.qcow2", nil
|
||||
//}
|
||||
version := codename
|
||||
floatVersion, err := strconv.ParseFloat(version, 64)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
majorVersion := int(math.Floor(floatVersion))
|
||||
filename := fmt.Sprintf("AlmaLinux-%d-GenericCloud-latest.x86_64.qcow2", majorVersion)
|
||||
url := fmt.Sprintf("https://mirror.csclub.uwaterloo.ca/almalinux/%s/cloud/x86_64/images/%s", version, filename)
|
||||
return mgr.DownloadTemplateGeneric(filename, url)
|
||||
}
|
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) GetIpv6ScriptsTemplateData() map[string]string {
|
||||
return map[string]string{
|
||||
"networkService": "NetworkManager.service",
|
||||
"networkTarget": "network-online.target",
|
||||
}
|
||||
}
|
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) addCloudInitSnippet(handle *guestfs.Guestfs) error {
|
||||
path := "/etc/cloud/cloud.cfg.d/99_csclub.cfg"
|
||||
mgr.logger.Debug().Msg("Writing to " + path)
|
||||
return handle.Write(path, getResource("fedora-cloud-init"))
|
||||
}
|
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) HasSELinuxEnabled() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
var almaLinuxYumRepoBaseUrlPattern *regexp.Regexp = regexp.MustCompile(
|
||||
"^(?P<scheme>https?://)[A-Za-z0-9./-]+(?P<path>/almalinux/\\$releasever/[A-Za-z0-9./$-]+)$",
|
||||
)
|
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) transformAlmaLinuxYumRepoBaseUrl(url string) string {
|
||||
submatches := almaLinuxYumRepoBaseUrlPattern.FindStringSubmatch(url)
|
||||
if submatches != nil {
|
||||
scheme, path := submatches[1], submatches[2]
|
||||
url = scheme + mgr.cfg.MirrorHost + path
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) PerformDistroSpecificModifications(handle *guestfs.Guestfs) (err error) {
|
||||
if err = mgr.setChronyOptions(handle, "/etc/chrony.conf"); err != nil {
|
||||
return
|
||||
}
|
||||
if err = mgr.setNetworkManagerOptions(handle); err != nil {
|
||||
return
|
||||
}
|
||||
if err = mgr.addCloudInitSnippet(handle); err != nil {
|
||||
return
|
||||
}
|
||||
if err = mgr.replaceYumRepoMirrorUrls(handle, mgr.transformAlmaLinuxYumRepoBaseUrl); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,161 @@
|
|||
package distros
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"golang.org/x/net/html"
|
||||
"libguestfs.org/guestfs"
|
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config"
|
||||
)
|
||||
|
||||
type FedoraTemplateManager struct {
|
||||
TemplateManager
|
||||
}
|
||||
|
||||
// NewFedoraTemplateManager returns a FedoraTemplateManager with the given
|
||||
// config values.
|
||||
func NewFedoraTemplateManager(cfg *config.Config) *FedoraTemplateManager {
|
||||
logger := log.With().Str("distro", "fedora").Logger()
|
||||
fedoraTemplateManager := FedoraTemplateManager{
|
||||
TemplateManager{
|
||||
cfg: cfg,
|
||||
logger: &logger,
|
||||
impl: nil,
|
||||
},
|
||||
}
|
||||
fedoraTemplateManager.TemplateManager.impl = &fedoraTemplateManager
|
||||
return &fedoraTemplateManager
|
||||
}
|
||||
|
||||
var numberSlashPattern *regexp.Regexp = regexp.MustCompile("^\\d+/$")
|
||||
|
||||
func isHyperlink(node *html.Node) bool {
|
||||
return node.Type == html.ElementNode && node.Data == "a" &&
|
||||
node.FirstChild != nil && node.FirstChild.Type == html.TextNode
|
||||
}
|
||||
|
||||
func getMatchingLinkFromHtml(node *html.Node, pattern *regexp.Regexp) string {
|
||||
if isHyperlink(node) {
|
||||
text := node.FirstChild.Data
|
||||
return pattern.FindString(text)
|
||||
}
|
||||
for c := node.FirstChild; c != nil; c = c.NextSibling {
|
||||
result := getMatchingLinkFromHtml(c, pattern)
|
||||
if result != "" {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func getMaxVersionFromHtml(node *html.Node, maxVersion *int64) {
|
||||
if isHyperlink(node) {
|
||||
text := node.FirstChild.Data
|
||||
if numberSlashPattern.FindString(text) != "" {
|
||||
version, _ := strconv.ParseInt(text[:len(text)-1], 10, 64)
|
||||
if version > *maxVersion {
|
||||
*maxVersion = version
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
for c := node.FirstChild; c != nil; c = c.NextSibling {
|
||||
getMaxVersionFromHtml(c, maxVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func (mgr *FedoraTemplateManager) GetLatestVersion() (version string, codename string, err error) {
|
||||
resp, err := http.Get("https://mirror.csclub.uwaterloo.ca/fedora/linux/releases/")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
node, err := html.Parse(resp.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var intVersion int64
|
||||
getMaxVersionFromHtml(node, &intVersion)
|
||||
if intVersion == 0 {
|
||||
err = errors.New("Could not determine latest Fedora version from HTML")
|
||||
} else {
|
||||
version = strconv.FormatInt(intVersion, 10)
|
||||
// Fedora doesn't have codenames, only numbered versions
|
||||
codename = version
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (mgr *FedoraTemplateManager) DownloadTemplate(codename string) (path string, err error) {
|
||||
version := codename
|
||||
pattern := regexp.MustCompile(fmt.Sprintf("^Fedora-Cloud-Base-%s-\\d+(\\.\\d+)?\\.x86_64\\.qcow2$", version))
|
||||
imagesUrl := fmt.Sprintf("https://mirror.csclub.uwaterloo.ca/fedora/linux/releases/%s/Cloud/x86_64/images/", version)
|
||||
resp, err := http.Get(imagesUrl)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
node, err := html.Parse(resp.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
filename := getMatchingLinkFromHtml(node, pattern)
|
||||
if filename == "" {
|
||||
err = errors.New("Could not find hyperlink for Fedora Cloud image")
|
||||
return
|
||||
}
|
||||
url := imagesUrl + filename
|
||||
return mgr.DownloadTemplateGeneric(filename, url)
|
||||
}
|
||||
|
||||
func (mgr *FedoraTemplateManager) GetIpv6ScriptsTemplateData() map[string]string {
|
||||
return map[string]string{
|
||||
"networkService": "NetworkManager.service",
|
||||
"networkTarget": "network-online.target",
|
||||
}
|
||||
}
|
||||
|
||||
func (mgr *FedoraTemplateManager) addCloudInitSnippet(handle *guestfs.Guestfs) error {
|
||||
path := "/etc/cloud/cloud.cfg.d/99_csclub.cfg"
|
||||
mgr.logger.Debug().Msg("Writing to " + path)
|
||||
return handle.Write(path, getResource("fedora-cloud-init"))
|
||||
}
|
||||
|
||||
func (mgr *FedoraTemplateManager) HasSELinuxEnabled() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
var fedoraYumRepoBaseUrlPattern *regexp.Regexp = regexp.MustCompile(
|
||||
"^(?P<scheme>https?://)[A-Za-z0-9./-]+(?P<path>/fedora/linux/[A-Za-z0-9./$-]+)$",
|
||||
)
|
||||
|
||||
func (mgr *FedoraTemplateManager) transformFedoraYumRepoBaseUrl(url string) string {
|
||||
submatches := fedoraYumRepoBaseUrlPattern.FindStringSubmatch(url)
|
||||
if submatches != nil {
|
||||
scheme, path := submatches[1], submatches[2]
|
||||
url = scheme + mgr.cfg.MirrorHost + path
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
func (mgr *FedoraTemplateManager) PerformDistroSpecificModifications(handle *guestfs.Guestfs) (err error) {
|
||||
if err = mgr.setChronyOptions(handle, "/etc/chrony.conf"); err != nil {
|
||||
return
|
||||
}
|
||||
if err = mgr.setNetworkManagerOptions(handle); err != nil {
|
||||
return
|
||||
}
|
||||
if err = mgr.addCloudInitSnippet(handle); err != nil {
|
||||
return
|
||||
}
|
||||
if err = mgr.replaceYumRepoMirrorUrls(handle, mgr.transformFedoraYumRepoBaseUrl); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
nmcli -f name -terse c | grep "^cloud-init e" | \
|
||||
while read conn_name; do
|
||||
iface=$(echo "$conn_name" | cut -d ' ' -f 2)
|
||||
nmcli c modify "$conn_name" ipv6.method link-local
|
||||
nmcli d reapply $iface
|
||||
done
|
||||
|
||||
ip -6 -brief addr show dynamic | \
|
||||
while read iface _ addrs; do
|
||||
for addr in $addrs; do
|
||||
ip addr del dev $iface $addr
|
||||
done
|
||||
done
|
||||
|
||||
ip -6 route show proto ra | \
|
||||
while read line; do
|
||||
ip route del $line
|
||||
done
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
#!/bin/bash
|
||||
set -ex
|
||||
INTERFACE=
|
||||
IPV4_ADDRESS=
|
||||
while read interface _ address; do
|
||||
if ! echo $address | grep -q '^172\.19\.134\.'; then
|
||||
continue
|
||||
fi
|
||||
INTERFACE=$interface
|
||||
IPV4_ADDRESS=$address
|
||||
break
|
||||
done < <(ip -4 -brief addr show)
|
||||
if [ -z "$INTERFACE" ]; then
|
||||
echo "Could not find primary interface" >&2
|
||||
exit 1
|
||||
fi
|
||||
NUM=$(echo $IPV4_ADDRESS | grep -oP '^172\.19\.134\.\K(\d+)')
|
||||
IPV6_ADDRESS="2620:101:f000:4903::$NUM/64"
|
||||
ip -6 addr add $IPV6_ADDRESS dev $INTERFACE
|
||||
ip -6 route add default via 2620:101:f000:4903::1 dev $INTERFACE
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
#server ntp.csclub.uwaterloo.ca
|
||||
server 129.97.167.12
|
||||
#server ntp.student.cs.uwaterloo.ca
|
||||
server 129.97.167.4
|
||||
#server ntp.cs.uwaterloo.ca
|
||||
server 129.97.15.14
|
||||
#server ntp.cscf.uwaterloo.ca
|
||||
server 129.97.15.15
|
||||
|
|
@ -0,0 +1 @@
|
|||
manage_etc_hosts: true
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
#!/bin/bash
|
||||
set -ex
|
||||
while read interface _; do
|
||||
if [ $interface = lo ]; then
|
||||
continue
|
||||
fi
|
||||
sysctl net.ipv6.conf.$interface.accept_ra=0
|
||||
sysctl net.ipv6.conf.$interface.accept_dad=0
|
||||
done < <(ip -brief link show)
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
|
||||
UNIVERSITY OF WATERLOO COMPUTER SCIENCE CLUB
|
||||
Welcome to the CSC cloud! :)
|
||||
This service was made possible thanks to donations from the Mathematics
|
||||
Endowment Fund (MEF) and the Computer Science Computing Facility (CSCF).
|
||||
|
||||
Documentation:
|
||||
https://docs.cloud.csclub.uwaterloo.ca
|
||||
Machine usage agreement:
|
||||
https://csclub.uwaterloo.ca/resources/machine-usage-agreement
|
||||
Questions/concerns can be emailed to:
|
||||
syscom@csclub.uwaterloo.ca
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
[main]
|
||||
dhcp = dhclient
|
||||
dns = none
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
search csclub.uwaterloo.ca uwaterloo.ca
|
||||
options rotate timeout:1 attempts:1 ndots:2
|
||||
|
||||
# CSC Nameservers
|
||||
nameserver 2620:101:f000:4901:c5c::4
|
||||
nameserver 2620:101:f000:7300:c5c::20
|
||||
nameserver 129.97.134.4
|
||||
nameserver 129.97.18.20
|
||||
|
||||
# IST Anycast (fallback)
|
||||
#nameserver 129.97.2.1
|
||||
#nameserver 129.97.2.2)
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
net.ipv6.conf.all.accept_ra = 0
|
||||
net.ipv6.conf.default.accept_ra = 0
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
[Time]
|
||||
# ntp.csclub.uwaterloo.ca ntp.student.cs.uwaterloo.ca
|
||||
NTP=129.97.167.12 129.97.167.4
|
||||
# ntp.cs.uwaterloo.ca ntp.cscf.uwaterloo.ca
|
||||
FallbackNTP=129.97.15.14 129.97.15.15
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
network:
|
||||
version: 2
|
||||
ethernets:
|
||||
id0:
|
||||
match:
|
||||
name: e*
|
||||
dhcp4: true
|
||||
accept-ra: false
|
||||
|
||||
apt_preserve_sources_list: true
|
||||
manage_etc_hosts: true
|
||||
|
|
@ -0,0 +1,526 @@
|
|||
package distros
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"libguestfs.org/guestfs"
|
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config"
|
||||
)
|
||||
|
||||
//go:embed resources
|
||||
var res embed.FS
|
||||
|
||||
func getResource(filename string) []byte {
|
||||
data, err := res.ReadFile("resources/" + filename)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func getTemplateResource(filename string) *template.Template {
|
||||
tmpl, err := template.ParseFS(res, "resources/"+filename)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return tmpl
|
||||
}
|
||||
|
||||
// A TemplateManager downloads and modifies VM templates for a distro.
|
||||
type TemplateManager struct {
|
||||
cfg *config.Config
|
||||
logger *zerolog.Logger
|
||||
impl IDistroSpecificTemplateManager
|
||||
}
|
||||
|
||||
type ITemplateManager interface {
|
||||
// GetLatestVersion returns the version number and codename of the
|
||||
// latest version of particular OS (e.g. version="22.04", codename="jammy")
|
||||
GetLatestVersion() (version string, codename string, err error)
|
||||
// DownloadTemplate downloads a VM template for a given OS codename
|
||||
// and returns the path to where it was downloaded
|
||||
DownloadTemplate(codename string) (path string, err error)
|
||||
// ModifyTemplate makes custom modifications to the downloaded VM template
|
||||
ModifyTemplate(filename string) error
|
||||
}
|
||||
|
||||
// An IDistroSpecificTemplateManager performs the distro-specific tasks
|
||||
// when modifying a VM template. It is used by the generic TemplateManager.
|
||||
type IDistroSpecificTemplateManager interface {
|
||||
// GetIpv6ScriptsTemplateData returns template data used by the
|
||||
// IPv6 scripts. The required keys are "networkService" and
|
||||
// "networkTarget"; the values should be a systemd service and target,
|
||||
// respectively.
|
||||
GetIpv6ScriptsTemplateData() map[string]string
|
||||
// PerformDistroSpecificModifications is called after
|
||||
// performDistroAgnosticModifications to modify a template in a
|
||||
// distro-specific way.
|
||||
PerformDistroSpecificModifications(handle *guestfs.Guestfs) error
|
||||
// HasSELinuxEnabled returns whether SELinux is enabled for this distro
|
||||
HasSELinuxEnabled() bool
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) DownloadTemplateGeneric(filename, url string) (path string, err error) {
|
||||
mgr.logger.Debug().Str("url", url).Msg("Downloading template")
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
// TODO: download to somewhere in /tmp
|
||||
path = filename
|
||||
out, err := os.Create(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer out.Close()
|
||||
_, err = io.Copy(out, resp.Body)
|
||||
return
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) performDistroAgnosticModifications(handle *guestfs.Guestfs) (err error) {
|
||||
templateData := mgr.impl.GetIpv6ScriptsTemplateData()
|
||||
if err = mgr.setupIpv6Scripts(handle, templateData); err != nil {
|
||||
return
|
||||
}
|
||||
if err = mgr.setResolvConf(handle); err != nil {
|
||||
return
|
||||
}
|
||||
if err = mgr.setMotd(handle); err != nil {
|
||||
return
|
||||
}
|
||||
if err = mgr.updateSshdConfig(handle); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) ModifyTemplate(filename string) (err error) {
|
||||
handle, err := mgr.getGuestfsMountedHandle(filename)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer mgr.unmountAndCloseGuestfsHandle(handle)
|
||||
if err = mgr.createAugeasHandle(handle); err != nil {
|
||||
return
|
||||
}
|
||||
defer mgr.closeAugeasHandle(handle)
|
||||
if err = mgr.performDistroAgnosticModifications(handle); err != nil {
|
||||
return
|
||||
}
|
||||
if err = mgr.impl.PerformDistroSpecificModifications(handle); err != nil {
|
||||
return
|
||||
}
|
||||
if mgr.impl.HasSELinuxEnabled() {
|
||||
if err = mgr.selinuxRelabelDirectories(handle); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
return mgr.saveAugeasValues(handle)
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) getGuestfsMountedHandle(filename string) (handle *guestfs.Guestfs, err error) {
|
||||
log := mgr.logger
|
||||
handle, err = guestfs.Create()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
log.Debug().Msg("Adding drive " + filename)
|
||||
if err = handle.Add_drive(filename, nil); err != nil {
|
||||
return
|
||||
}
|
||||
log.Debug().Msg("Launching VM")
|
||||
if err = handle.Launch(); err != nil {
|
||||
return
|
||||
}
|
||||
log.Debug().Msg("Inspecting OS")
|
||||
partitions, err := handle.Inspect_os()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if len(partitions) != 1 {
|
||||
return nil, errors.New(fmt.Sprintf("Expected 1 root partition, found %d", len(partitions)))
|
||||
}
|
||||
rootPartition := partitions[0]
|
||||
log.Debug().Msg(fmt.Sprintf("Mounting root filesystem %s on /", rootPartition))
|
||||
if err = handle.Mount(rootPartition, "/"); err != nil {
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) unmountGuestfsDrive(handle *guestfs.Guestfs) {
|
||||
if err := handle.Umount("/", nil); err != nil {
|
||||
mgr.logger.Error().Err(err).Msg("")
|
||||
}
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) unmountAndCloseGuestfsHandle(handle *guestfs.Guestfs) {
|
||||
mgr.unmountGuestfsDrive(handle)
|
||||
handle.Close()
|
||||
}
|
||||
|
||||
func getSelinuxType(handle *guestfs.Guestfs) (selinuxType string, err error) {
|
||||
lines, err := handle.Grep("^SELINUXTYPE=", "/etc/selinux/config", nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if len(lines) != 1 {
|
||||
err = errors.New(fmt.Sprintf("Expected 1 line containing SELINUXTYPE, found %d", len(lines)))
|
||||
return
|
||||
}
|
||||
selinuxType = strings.Split(lines[0], "=")[1]
|
||||
return
|
||||
}
|
||||
|
||||
func getSelinuxDefaultSpecfile(handle *guestfs.Guestfs) (specfile string, err error) {
|
||||
selinuxType, err := getSelinuxType(handle)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
specfile = fmt.Sprintf("/etc/selinux/%s/contexts/files/file_contexts", selinuxType)
|
||||
return
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) selinuxRelabel(handle *guestfs.Guestfs, specfile, dir string) (err error) {
|
||||
mgr.logger.Debug().
|
||||
Str("specfile", specfile).
|
||||
Str("path", dir).
|
||||
Msg("Relabeling SELinux security context")
|
||||
return handle.Selinux_relabel(specfile, dir, nil)
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) selinuxRelabelDirectories(handle *guestfs.Guestfs) (err error) {
|
||||
specfile, err := getSelinuxDefaultSpecfile(handle)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, dir := range []string{"/etc", "/var/lib/cloud"} {
|
||||
if err = mgr.selinuxRelabel(handle, specfile, dir); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// setChronyOptions sets custom NTP server URLs in a chrony config file.
|
||||
// It assumes that a line beginning with "pool" will already be present.
|
||||
func (mgr *TemplateManager) setChronyOptions(handle *guestfs.Guestfs, path string) (err error) {
|
||||
oldLines, err := handle.Read_lines(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
snippet := string(getResource("chrony-snippet"))
|
||||
snippetLines := strings.Split(snippet, "\n")
|
||||
wroteSnippet := false
|
||||
newLines := make([]string, 0, len(oldLines)+len(snippetLines))
|
||||
for _, line := range oldLines {
|
||||
if strings.HasPrefix(line, "pool ") {
|
||||
newLines = append(newLines, "#"+line)
|
||||
if !wroteSnippet {
|
||||
newLines = append(newLines, snippetLines...)
|
||||
wroteSnippet = true
|
||||
}
|
||||
} else {
|
||||
newLines = append(newLines, line)
|
||||
}
|
||||
}
|
||||
newContent := strings.Join(newLines, "\n")
|
||||
mgr.logger.Debug().Msg("Writing new content to " + path)
|
||||
return handle.Write(path, []byte(newContent))
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) setNetworkManagerOptions(handle *guestfs.Guestfs) (err error) {
|
||||
if err = handle.Mkdir_p("/etc/NetworkManager/conf.d"); err != nil {
|
||||
return
|
||||
}
|
||||
path := "/etc/NetworkManager/conf.d/99_csclub.conf"
|
||||
mgr.logger.Debug().Msg("Writing to " + path)
|
||||
return handle.Write(path, getResource("network-manager-snippet"))
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) setupIpv6Scripts(handle *guestfs.Guestfs, templateData map[string]string) (err error) {
|
||||
log := mgr.logger
|
||||
scripts := []string{"99_csclub_ipv6_addr.sh"}
|
||||
if templateData["networkService"] == "NetworkManager.service" {
|
||||
scripts = append(scripts, "98_csclub_disable_nm_ipv6.sh")
|
||||
}
|
||||
scriptDir := "/var/lib/cloud/scripts/per-boot"
|
||||
if err = handle.Mkdir_p(scriptDir); err != nil {
|
||||
return
|
||||
}
|
||||
for _, filename := range scripts {
|
||||
path := scriptDir + "/" + filename
|
||||
log.Debug().Msg("Writing to " + path)
|
||||
if err = handle.Write(path, getResource(filename)); err != nil {
|
||||
return
|
||||
}
|
||||
if err = handle.Chmod(0755, path); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
sysctlPath := "/etc/sysctl.d/csclub.conf"
|
||||
log.Debug().Msg("Writing to " + sysctlPath)
|
||||
return handle.Write(sysctlPath, getResource("sysctl.conf"))
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) setResolvConf(handle *guestfs.Guestfs) (err error) {
|
||||
mgr.logger.Debug().Msg("Writing to /etc/resolv.conf")
|
||||
if err = handle.Rm_f("/etc/resolv.conf"); err != nil {
|
||||
return
|
||||
}
|
||||
return handle.Write("/etc/resolv.conf", getResource("resolv.conf"))
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) createAugeasHandle(handle *guestfs.Guestfs) (err error) {
|
||||
mgr.logger.Debug().Msg("Creating a new Augeas handle")
|
||||
return handle.Aug_init("/", 0)
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) closeAugeasHandle(handle *guestfs.Guestfs) {
|
||||
if err := handle.Aug_close(); err != nil {
|
||||
mgr.logger.Error().Err(err).Msg("")
|
||||
}
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) saveAugeasValues(handle *guestfs.Guestfs) error {
|
||||
mgr.logger.Debug().Msg("Saving Augeas values")
|
||||
return handle.Aug_save()
|
||||
}
|
||||
|
||||
// requires an Augeas handle to be open
|
||||
func (mgr *TemplateManager) setDhclientOptions(handle *guestfs.Guestfs) (err error) {
|
||||
mgr.logger.Debug().Msg("Retrieving dhclient request options")
|
||||
dhclientRequestOptionNodes, err := handle.Aug_ls("/files/etc/dhcp/dhclient.conf/request")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
optionsToRemove := map[string]bool{
|
||||
"domain-name": true,
|
||||
"domain-name-servers": true,
|
||||
"domain-search": true,
|
||||
"dhcp6.name-servers": true,
|
||||
"dhcp6.domain-search": true,
|
||||
"dhcp6.fqdn": true,
|
||||
"dhcp6.sntp-servers": true,
|
||||
"ntp-servers": true,
|
||||
"netbios-name-servers": true,
|
||||
}
|
||||
for _, optionPath := range dhclientRequestOptionNodes {
|
||||
// optionPath will look something like /files/etc/dhcp/dhclient.conf/request/5
|
||||
var optionValue string
|
||||
optionValue, err = handle.Aug_get(optionPath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, ok := optionsToRemove[optionValue]; !ok {
|
||||
continue
|
||||
}
|
||||
mgr.logger.Debug().Msg("Removing dhclient request option " + optionValue)
|
||||
if _, err = handle.Aug_rm(optionPath); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func getNamedRegexGroup(re *regexp.Regexp, submatches []string, groupName string) string {
|
||||
var value string
|
||||
for i, subexpName := range re.SubexpNames() {
|
||||
if subexpName == groupName {
|
||||
value = submatches[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if value == "" {
|
||||
panic("Could not find regex group " + groupName)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) addAugeasComment(
|
||||
handle *guestfs.Guestfs, numExistingComments int, parentNode string, comment string,
|
||||
) error {
|
||||
return handle.Aug_set(
|
||||
parentNode+fmt.Sprintf("/#comment[%d]", numExistingComments),
|
||||
comment,
|
||||
)
|
||||
}
|
||||
|
||||
// replaceYumRepoMirrorUrls comments out the metalink URLs and uncomments and
|
||||
// replaces the baseurl URLs for each repo in /etc/yum.repos.d.
|
||||
// It assumes that the baseurl will be a commented line.
|
||||
//
|
||||
// transformBaseUrl accepts a repo base URL and replaces the host (and
|
||||
// optionally parts of the path) as appropriate. It should return the input
|
||||
// if the URL should not be modified.
|
||||
func (mgr *TemplateManager) replaceYumRepoMirrorUrls(
|
||||
handle *guestfs.Guestfs, transformBaseurl func(string) string,
|
||||
) (err error) {
|
||||
log := mgr.logger
|
||||
repoPaths, err := handle.Aug_ls("/files/etc/yum.repos.d")
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not enumerate yum repos: %w", err)
|
||||
}
|
||||
// A repoPath looks like e.g. /files/etc/yum.repos.d/almalinux-appstream.repo
|
||||
for _, repoPath := range repoPaths {
|
||||
subrepoPaths, err := handle.Aug_ls(repoPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not enumerate subrepos for %s: %w", repoPath, err)
|
||||
}
|
||||
// A subrepoPath looks like e.g. /files/etc/yum.repos.d/almalinux-appstream.repo/appstream
|
||||
for _, subrepoPath := range subrepoPaths {
|
||||
keyPaths, err := handle.Aug_ls(subrepoPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not enumerate keys for %s: %w", subrepoPath, err)
|
||||
}
|
||||
var (
|
||||
numComments int
|
||||
commentedBaseurl string
|
||||
mirrorlist string
|
||||
uncommentedBaseurl string
|
||||
)
|
||||
// A keyPath looks like e.g. /files/etc/yum.repos.d/almalinux-appstream.repo/appstream/mirrorlist
|
||||
for _, keyPath := range keyPaths {
|
||||
// extract the last part of the node path
|
||||
key, err := handle.Aug_label(keyPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not get label of %s: %w", keyPath, err)
|
||||
}
|
||||
value, err := handle.Aug_get(keyPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Could not get %s: %w", keyPath, err)
|
||||
}
|
||||
if strings.HasPrefix(key, "#comment") {
|
||||
numComments += 1
|
||||
if strings.HasPrefix(value, "baseurl=") {
|
||||
commentedBaseurl = strings.Split(value, "=")[1]
|
||||
}
|
||||
} else if key == "mirrorlist" {
|
||||
mirrorlist = value
|
||||
} else if key == "baseurl" {
|
||||
uncommentedBaseurl = value
|
||||
}
|
||||
}
|
||||
var baseurl string
|
||||
if uncommentedBaseurl != "" {
|
||||
baseurl = transformBaseurl(uncommentedBaseurl)
|
||||
if baseurl == uncommentedBaseurl {
|
||||
baseurl = ""
|
||||
}
|
||||
} else if commentedBaseurl != "" {
|
||||
baseurl = transformBaseurl(commentedBaseurl)
|
||||
if baseurl == commentedBaseurl {
|
||||
baseurl = ""
|
||||
}
|
||||
}
|
||||
if baseurl != "" {
|
||||
log.Debug().Msg(fmt.Sprintf("Setting %s to %s", subrepoPath+"/baseurl", baseurl))
|
||||
if err = handle.Aug_set(subrepoPath+"/baseurl", baseurl); err != nil {
|
||||
return fmt.Errorf("Could not set baseurl for %s: %w", subrepoPath, err)
|
||||
}
|
||||
if mirrorlist != "" {
|
||||
// comment out the mirrorlist line
|
||||
log.Debug().Msg("Commenting out " + subrepoPath + "/mirrorlist")
|
||||
if _, err = handle.Aug_rm(subrepoPath + "/mirrorlist"); err != nil {
|
||||
return fmt.Errorf("Could not remove %s: %w", subrepoPath+"/mirrorlist", err)
|
||||
}
|
||||
if err = mgr.addAugeasComment(handle, numComments, subrepoPath, "mirrorlist="+mirrorlist); err != nil {
|
||||
return fmt.Errorf("Could not insert comment in %s: %w", subrepoPath, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// requires an Augeas handle to be open
|
||||
func (mgr *TemplateManager) replaceDebianMirrorUrls(handle *guestfs.Guestfs) (err error) {
|
||||
log := mgr.logger
|
||||
// Some Augeas nodes under /files/etc/apt/sources.list are comments,
|
||||
// so we use /*/uri to make sure that we only get the actual entries
|
||||
sourcesListEntries, err := handle.Aug_match("/files/etc/apt/sources.list/*/uri")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, uriPath := range sourcesListEntries {
|
||||
var (
|
||||
uriValue string
|
||||
parsedUrl *url.URL
|
||||
distValue string
|
||||
typeValue string
|
||||
)
|
||||
if uriValue, err = handle.Aug_get(uriPath); err != nil {
|
||||
return
|
||||
}
|
||||
parsedUrl, err = url.Parse(uriValue)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
parsedUrl.Host = mgr.cfg.MirrorHost
|
||||
newUriValue := parsedUrl.String()
|
||||
// strip off the "/uri" from the node path
|
||||
entryPath := uriPath[:len(uriPath)-4]
|
||||
typePath := entryPath + "/type"
|
||||
if typeValue, err = handle.Aug_get(typePath); err != nil {
|
||||
return
|
||||
}
|
||||
distPath := entryPath + "/distribution"
|
||||
if distValue, err = handle.Aug_get(distPath); err != nil {
|
||||
return
|
||||
}
|
||||
if typeValue == "deb-src" {
|
||||
log.Debug().
|
||||
Str("URL", uriValue).
|
||||
Str("distribution", distValue).
|
||||
Msg("Removing deb-src entry")
|
||||
if _, err = handle.Aug_rm(entryPath); err != nil {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if uriValue == newUriValue {
|
||||
continue
|
||||
}
|
||||
log.Debug().
|
||||
Str("distribution", distValue).
|
||||
Str("oldURL", uriValue).
|
||||
Str("newURL", newUriValue).
|
||||
Msg("Replacing URL in sources.list")
|
||||
if err = handle.Aug_set(uriPath, newUriValue); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// requires an Augeas handle to be open
|
||||
func (mgr *TemplateManager) updateSshdConfig(handle *guestfs.Guestfs) error {
|
||||
mgr.logger.Debug().Msg("Setting PrintLastLog=no in sshd_config")
|
||||
return handle.Aug_set("/files/etc/ssh/sshd_config/PrintLastLog", "no")
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) setTimesyncdConf(handle *guestfs.Guestfs) (err error) {
|
||||
mgr.logger.Debug().Msg("Writing custom timesyncd.conf")
|
||||
if err = handle.Mkdir_p("/etc/systemd/timesyncd.conf.d"); err != nil {
|
||||
return
|
||||
}
|
||||
return handle.Write("/etc/systemd/timesyncd.conf.d/csclub.conf", getResource("timesyncd.conf"))
|
||||
}
|
||||
|
||||
func (mgr *TemplateManager) setMotd(handle *guestfs.Guestfs) error {
|
||||
mgr.logger.Debug().Msg("Writing to /etc/motd")
|
||||
return handle.Write("/etc/motd", getResource("motd"))
|
||||
}
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
package distros
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"libguestfs.org/guestfs"
|
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config"
|
||||
)
|
||||
|
||||
type UbuntuTemplateManager struct {
|
||||
TemplateManager
|
||||
}
|
||||
|
||||
// NewUbuntuTemplateManager returns a UbuntuTemplateManager with the given
|
||||
// config values.
|
||||
func NewUbuntuTemplateManager(cfg *config.Config) *UbuntuTemplateManager {
|
||||
logger := log.With().Str("distro", "ubuntu").Logger()
|
||||
// The child embeds the parent, and the parent has a reference to the child
|
||||
ubuntuTemplateManager := UbuntuTemplateManager{
|
||||
TemplateManager{
|
||||
cfg: cfg,
|
||||
logger: &logger,
|
||||
impl: nil,
|
||||
},
|
||||
}
|
||||
ubuntuTemplateManager.TemplateManager.impl = &ubuntuTemplateManager
|
||||
return &ubuntuTemplateManager
|
||||
}
|
||||
|
||||
type UbuntuVersions struct {
|
||||
Products map[string]struct {
|
||||
Arch string `json:"arch"`
|
||||
Release string `json:"release"`
|
||||
ReleaseTitle string `json:"release_title"`
|
||||
Version string `json:"version"`
|
||||
} `json:"products"`
|
||||
}
|
||||
|
||||
// GetLatestVersion returns the version and codename of the latest Ubuntu release.
|
||||
// e.g. version = 22.04, codename = jammy
|
||||
func (mgr *UbuntuTemplateManager) GetLatestVersion() (version string, codename string, err error) {
|
||||
resp, err := http.Get("https://cloud-images.ubuntu.com/releases/streams/v1/com.ubuntu.cloud:released:download.json")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
buf, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var data UbuntuVersions
|
||||
if err = json.Unmarshal(buf, &data); err != nil {
|
||||
return
|
||||
}
|
||||
for _, val := range data.Products {
|
||||
if val.Arch == "amd64" && strings.HasSuffix(val.ReleaseTitle, "LTS") && val.Version > version {
|
||||
version = val.Version
|
||||
codename = val.Release
|
||||
}
|
||||
}
|
||||
if version == "" {
|
||||
err = errors.New("Could not find latest LTS version of Ubuntu")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DownloadTemplate downloads the cloud image for the given Ubuntu codename,
|
||||
// and returns the filename to which it was written.
|
||||
func (mgr *UbuntuTemplateManager) DownloadTemplate(codename string) (path string, err error) {
|
||||
filename := fmt.Sprintf("%s-server-cloudimg-amd64-disk-kvm.img", codename)
|
||||
url := fmt.Sprintf("https://cloud-images.ubuntu.com/%s/current/%s", codename, filename)
|
||||
return mgr.DownloadTemplateGeneric(filename, url)
|
||||
}
|
||||
|
||||
func (mgr *UbuntuTemplateManager) addCloudInitSnippet(handle *guestfs.Guestfs) error {
|
||||
path := "/etc/cloud/cloud.cfg.d/99_csclub.cfg"
|
||||
mgr.logger.Debug().Msg("Writing to " + path)
|
||||
return handle.Write(path, getResource("ubuntu-cloud-init"))
|
||||
}
|
||||
|
||||
func (mgr *UbuntuTemplateManager) disableNoisyMotdMessages(handle *guestfs.Guestfs) {
|
||||
log := mgr.logger
|
||||
// Setting these files to non-executable (chmod a-x) disables them
|
||||
filesToDisable := []string{
|
||||
"00-header",
|
||||
"10-help-text",
|
||||
"50-landscape-sysinfo",
|
||||
"50-motd-news",
|
||||
"88-esm-announce",
|
||||
}
|
||||
for _, filename := range filesToDisable {
|
||||
path := "/etc/update-motd.d/" + filename
|
||||
log.Debug().Msg("Disabling " + path)
|
||||
if err := handle.Chmod(0644, path); err != nil {
|
||||
log.Warn().Err(err).Msg("Could not disable " + path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (mgr *UbuntuTemplateManager) HasSELinuxEnabled() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (mgr *UbuntuTemplateManager) GetIpv6ScriptsTemplateData() map[string]string {
|
||||
return map[string]string{
|
||||
"networkService": "systemd-networkd.service",
|
||||
"networkTarget": "network.target",
|
||||
}
|
||||
}
|
||||
|
||||
func (mgr *UbuntuTemplateManager) PerformDistroSpecificModifications(handle *guestfs.Guestfs) (err error) {
|
||||
if err = mgr.setTimesyncdConf(handle); err != nil {
|
||||
return
|
||||
}
|
||||
if err = mgr.setDhclientOptions(handle); err != nil {
|
||||
return
|
||||
}
|
||||
if err = mgr.replaceDebianMirrorUrls(handle); err != nil {
|
||||
return
|
||||
}
|
||||
if err = mgr.addCloudInitSnippet(handle); err != nil {
|
||||
return
|
||||
}
|
||||
mgr.disableNoisyMotdMessages(handle)
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
# This worked on Debian bullseye on the CSC machines - will need to
|
||||
# be updated for other versions or other distros
|
||||
# TODO: automatically generate this list from the core dependencies
|
||||
# (golang-guestfs-dev, libvirt0, qemu-system-x86)
|
||||
dependencies=(
|
||||
golang-guestfs-dev
|
||||
libbrlapi0
|
||||
libcacard0
|
||||
libcapstone4
|
||||
libdaxctl1
|
||||
libfdt1
|
||||
libguestfs0
|
||||
libguestfs-dev
|
||||
libndctl6
|
||||
libpmem1
|
||||
libslirp0
|
||||
libspice-server1
|
||||
liburing1
|
||||
libusbredirparser1
|
||||
libvdeplug2
|
||||
libvirglrenderer1
|
||||
libvirt0
|
||||
libxencall1
|
||||
libxendevicemodel1
|
||||
libxenevtchn1
|
||||
libxenforeignmemory1
|
||||
libxengnttab1
|
||||
libxenmisc4
|
||||
libxenstore3
|
||||
libxentoolcore1
|
||||
libxentoollog1
|
||||
libyajl2
|
||||
qemu-system-x86
|
||||
qemu-utils
|
||||
seabios
|
||||
)
|
||||
|
||||
if [ $# -eq 1 ] && [ "$1" = guestfish ]; then
|
||||
dependencies=("${dependencies[@]}" libguestfs-tools libconfig9)
|
||||
fi
|
||||
|
||||
is_installed() {
|
||||
status=$(dpkg-query --show --showformat '${db:Status-Status}' "$1" 2>/dev/null)
|
||||
[ $? -eq 0 ] && [ "$status" = installed ]
|
||||
}
|
||||
|
||||
mkdir -p guestfs/deps
|
||||
cd guestfs
|
||||
|
||||
for dep in "${dependencies[@]}"; do
|
||||
set -x
|
||||
apt download $dep
|
||||
dpkg -x ${dep}_*.deb deps
|
||||
set +x
|
||||
done
|
||||
rm *.deb
|
||||
|
||||
go mod init guestfs
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#!/bin/sh
|
||||
|
||||
if [ -x /usr/bin/qemu-system-x86_64 ]; then
|
||||
exec /usr/bin/qemu-system-x86_64 "$@"
|
||||
fi
|
||||
DEPS_DIR=guestfs/deps
|
||||
exec $DEPS_DIR/usr/bin/qemu-system-x86_64 -L $DEPS_DIR/usr/share/seabios "$@"
|
||||
Loading…
Reference in New Issue