commit
08f7371198
@ -0,0 +1,3 @@ |
||||
/cloudbuild |
||||
/guestfs |
||||
*.swp |
||||
@ -0,0 +1,40 @@ |
||||
DEPS_DIR = guestfs/deps
|
||||
LIBRARY_PATH = $(DEPS_DIR)/usr/lib/x86_64-linux-gnu:$(DEPS_DIR)/lib/x86_64-linux-gnu
|
||||
LIBGUESTFS_PATH = guestfs/appliance
|
||||
LIBGUESTFS_HV = scripts/qemu.sh
|
||||
APPLIANCE_VERSION = 1.46.0
|
||||
DOWNLOAD_DEPS_ARGS =
|
||||
|
||||
# Export LIBGUESTFS_DEBUG=1 to debug
|
||||
|
||||
ifeq ($(GUESTFISH),1) |
||||
DOWNLOAD_DEPS_ARGS = guestfish
|
||||
endif |
||||
|
||||
all: |
||||
LIBRARY_PATH=$(LIBRARY_PATH) CGO_LDFLAGS='-l:libvirt.so.0 -l:libyajl.so.2' go build
|
||||
|
||||
run: |
||||
LD_LIBRARY_PATH=$(LIBRARY_PATH) LIBGUESTFS_PATH=$(LIBGUESTFS_PATH) LIBGUESTFS_HV=$(LIBGUESTFS_HV) LIBGUESTFS_BACKEND_SETTINGS=force_tcg ./cloudbuild
|
||||
|
||||
guestfish: |
||||
LD_LIBRARY_PATH=$(LIBRARY_PATH) LIBGUESTFS_PATH=$(LIBGUESTFS_PATH) LIBGUESTFS_HV=$(LIBGUESTFS_HV) LIBGUESTFS_BACKEND_SETTINGS=force_tcg PATH=guestfs/qemu-utils-deps/usr/bin:$(PATH) $(DEPS_DIR)/usr/bin/guestfish
|
||||
|
||||
deps: |
||||
scripts/download-deps.sh $(DOWNLOAD_DEPS_ARGS)
|
||||
|
||||
appliance-download: |
||||
cd guestfs && \
|
||||
wget https://download.libguestfs.org/binaries/appliance/appliance-$(APPLIANCE_VERSION).tar.xz && \
|
||||
tar Jxvf appliance-$(APPLIANCE_VERSION).tar.xz && \
|
||||
rm appliance-$(APPLIANCE_VERSION).tar.xz
|
||||
|
||||
appliance: $(DEPS_DIR)usr/bin/supermin |
||||
mkdir -p /var/tmp/.guestfs-`id -u`
|
||||
$(DEPS_DIR)usr/bin/supermin --build --verbose --if-newer --lock /var/tmp/.guestfs-`id -u`/lock --copy-kernel -f ext2 --host-cpu x86_64 $(DEPS_DIR)usr/lib/x86_64-linux-gnu/guestfs/supermin.d -o /var/tmp/.guestfs-`id -u`/appliance.d
|
||||
mv /var/tmp/.guestfs-`id -u`/appliance.d guestfs/appliance
|
||||
|
||||
$(DEPS_DIR)usr/bin/supermin: |
||||
cd guestfs && apt download supermin && dpkg -x supermin_*.deb deps && rm supermin_*.deb
|
||||
|
||||
.PHONY: all run guestfish deps appliance-download appliance |
||||
@ -0,0 +1,11 @@ |
||||
module git.csclub.uwaterloo.ca/cloud/cloudbuild |
||||
|
||||
go 1.17 |
||||
|
||||
replace libguestfs.org/guestfs => ./guestfs |
||||
|
||||
require ( |
||||
github.com/rs/zerolog v1.26.1 |
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d |
||||
libguestfs.org/guestfs v0.0.0-00010101000000-000000000000 |
||||
) |
||||
@ -0,0 +1,33 @@ |
||||
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= |
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= |
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= |
||||
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= |
||||
github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= |
||||
github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= |
||||
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= |
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= |
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= |
||||
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= |
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= |
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= |
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= |
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= |
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= |
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= |
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= |
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= |
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= |
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= |
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= |
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= |
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= |
||||
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= |
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= |
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= |
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= |
||||
@ -0,0 +1,26 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"os" |
||||
"time" |
||||
|
||||
"github.com/rs/zerolog" |
||||
"github.com/rs/zerolog/log" |
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/cloudbuilder" |
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" |
||||
) |
||||
|
||||
func setupLogging() { |
||||
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}) |
||||
} |
||||
|
||||
func main() { |
||||
setupLogging() |
||||
cfg := config.New() |
||||
builder := cloudbuilder.New(cfg) |
||||
|
||||
if err := builder.Start(); err != nil { |
||||
panic(err) |
||||
} |
||||
} |
||||
@ -0,0 +1,287 @@ |
||||
package cloudbuilder |
||||
|
||||
import ( |
||||
"bytes" |
||||
_ "embed" |
||||
"errors" |
||||
"fmt" |
||||
"net/smtp" |
||||
"os" |
||||
"path" |
||||
"regexp" |
||||
"strconv" |
||||
"strings" |
||||
"text/template" |
||||
"time" |
||||
|
||||
"github.com/rs/zerolog/log" |
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/cloudstack" |
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" |
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/distros" |
||||
) |
||||
|
||||
//go:embed email-message.txt
|
||||
var emailMessageTemplate string |
||||
|
||||
type CloudBuilder struct { |
||||
cfg *config.Config |
||||
client *cloudstack.CloudstackClient |
||||
} |
||||
|
||||
func New(cfg *config.Config) *CloudBuilder { |
||||
return &CloudBuilder{ |
||||
cfg: cfg, |
||||
client: cloudstack.New(cfg), |
||||
} |
||||
} |
||||
|
||||
func (c *CloudBuilder) moveTemplateToUploadArea(templatePath string) (newPath string, err error) { |
||||
newPath = path.Join(c.cfg.UploadDirectory, path.Base(templatePath)) |
||||
log.Debug().Msg(fmt.Sprintf("Moving %s to %s", templatePath, newPath)) |
||||
err = os.Rename(templatePath, newPath) |
||||
return |
||||
} |
||||
|
||||
func (c *CloudBuilder) UploadTemplate( |
||||
templatePath string, templateName string, osDescription string, |
||||
) (template *cloudstack.Template, err error) { |
||||
templatePath, err = c.moveTemplateToUploadArea(templatePath) |
||||
if err != nil { |
||||
return |
||||
} |
||||
osTypeId, err := c.client.GetOsTypeId(osDescription) |
||||
if err != nil { |
||||
return |
||||
} |
||||
uploadUrl := strings.TrimRight(c.cfg.UploadBaseUrl, "/") + "/" + path.Base(templatePath) |
||||
log.Debug().Msg(fmt.Sprintf("Uploading new template from %s", uploadUrl)) |
||||
template, err = c.client.RegisterTemplate(templateName, uploadUrl, osTypeId) |
||||
if err != nil { |
||||
return |
||||
} |
||||
const pollInterval = 5 * time.Second |
||||
// Due to clock drift, we may end up waiting a bit longer than this
|
||||
const maxTimeToWait = 5 * time.Minute |
||||
var timeWaited time.Duration = 0 |
||||
for !template.IsReady { |
||||
// TODO: make sure that DownloadDetails[0] exists
|
||||
log.Debug(). |
||||
Str("templateName", template.Name). |
||||
Str("status", template.Status). |
||||
Str("downloadPercent", template.DownloadDetails[0].DownloadPercent). |
||||
Str("downloadState", template.DownloadDetails[0].DownloadState). |
||||
Msg("Template is not ready yet...") |
||||
time.Sleep(pollInterval) |
||||
timeWaited += pollInterval |
||||
if timeWaited >= maxTimeToWait { |
||||
err = errors.New("Timed out waiting for template to be ready") |
||||
return |
||||
} |
||||
template, err = c.client.GetTemplate(template.Id) |
||||
if err != nil { |
||||
return |
||||
} |
||||
} |
||||
log.Debug().Msg("Deleting " + templatePath) |
||||
err = os.Remove(templatePath) |
||||
return |
||||
} |
||||
|
||||
func (c *CloudBuilder) CreateVM( |
||||
name string, templateID string, |
||||
) (virtualMachine *cloudstack.VirtualMachine, err error) { |
||||
vmID, err := c.client.DeployVirtualMachine(name, templateID) |
||||
if err != nil { |
||||
return |
||||
} |
||||
virtualMachine, err = c.client.GetVirtualMachine(vmID) |
||||
if err != nil { |
||||
return |
||||
} |
||||
const pollInterval = 10 * time.Second |
||||
// Due to clock drift, we may end up waiting a bit longer than this
|
||||
const maxTimeToWait = 30 * time.Minute |
||||
var timeWaited time.Duration = 0 |
||||
log.Debug().Str("templateID", templateID).Msg("Creating new VM") |
||||
for virtualMachine.State != "Running" { |
||||
log.Debug(). |
||||
Str("name", name). |
||||
Str("id", virtualMachine.Id). |
||||
Str("state", virtualMachine.State). |
||||
Msg("VM is not ready yet...") |
||||
time.Sleep(pollInterval) |
||||
timeWaited += pollInterval |
||||
if timeWaited >= maxTimeToWait { |
||||
err = errors.New("Timed out waiting for VM to be ready") |
||||
return |
||||
} |
||||
virtualMachine, err = c.client.GetVirtualMachine(vmID) |
||||
if err != nil { |
||||
return |
||||
} |
||||
} |
||||
log.Debug().Msg("VM successfully created") |
||||
return |
||||
} |
||||
|
||||
type DistroInfo struct { |
||||
name string |
||||
manager distros.ITemplateManager |
||||
osDescription string |
||||
user string |
||||
} |
||||
|
||||
func (c *CloudBuilder) getExistingTemplateVersions() map[string]string { |
||||
templatePattern := regexp.MustCompile("^CSC (?P<distro>[A-Za-z][A-Za-z ]+[A-Za-z]) (?P<version>\\d+(\\.\\d+)?)$") |
||||
mostRecentVersions := make(map[string]string) |
||||
templates := c.client.ListTemplates() |
||||
for _, template := range templates { |
||||
submatches := templatePattern.FindStringSubmatch(template.Name) |
||||
if submatches == nil { |
||||
log.Debug().Msg(fmt.Sprintf("Template '%s' did not match pattern, skipping", template.Name)) |
||||
continue |
||||
} |
||||
distro := submatches[1] |
||||
version := submatches[2] |
||||
log.Debug().Str("distro", distro).Str("version", version).Msg("Found template") |
||||
otherVersion, ok := mostRecentVersions[distro] |
||||
if !ok || version > otherVersion { |
||||
mostRecentVersions[distro] = version |
||||
} |
||||
} |
||||
return mostRecentVersions |
||||
} |
||||
|
||||
func (c *CloudBuilder) sendEmailNotification( |
||||
templateName string, vm *cloudstack.VirtualMachine, vmUser string, |
||||
) (err error) { |
||||
tmpl := template.Must(template.New("email-message").Parse(emailMessageTemplate)) |
||||
data := map[string]interface{}{ |
||||
"cfg": c.cfg, |
||||
"date": time.Now().Format(time.RFC1123Z), |
||||
"templateName": templateName, |
||||
"vm": vm, |
||||
"vmUser": vmUser, |
||||
} |
||||
var buf bytes.Buffer |
||||
if err = tmpl.Execute(&buf, data); err != nil { |
||||
return |
||||
} |
||||
// The lines of the body need to be CRLF terminated
|
||||
// See https://pkg.go.dev/net/smtp#SendMail
|
||||
// Make sure that email-message.txt uses Unix-style LF endings
|
||||
msg := bytes.ReplaceAll(buf.Bytes(), []byte("\n"), []byte("\r\n")) |
||||
log.Debug(). |
||||
Str("to", c.cfg.EmailRecipient). |
||||
Msg("sending email notification") |
||||
return smtp.SendMail( |
||||
c.cfg.EmailServer, |
||||
nil, /* auth */ |
||||
c.cfg.EmailSender, |
||||
[]string{c.cfg.EmailRecipient}, |
||||
msg, |
||||
) |
||||
} |
||||
|
||||
func (c *CloudBuilder) createNewTemplate( |
||||
newVersion, codename string, distroInfo *DistroInfo, |
||||
) (template *cloudstack.Template, err error) { |
||||
distroManager := distroInfo.manager |
||||
templatePath, err := distroManager.DownloadTemplate(codename) |
||||
if err != nil { |
||||
return |
||||
} |
||||
if err = distroManager.ModifyTemplate(templatePath); err != nil { |
||||
return |
||||
} |
||||
templateName := fmt.Sprintf("CSC %s %s", distroInfo.name, newVersion) |
||||
template, err = c.UploadTemplate(templatePath, templateName, distroInfo.osDescription) |
||||
return |
||||
} |
||||
|
||||
func (c *CloudBuilder) versionStringCompare(version1, version2 string) int { |
||||
f1, err := strconv.ParseFloat(version1, 32) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
f2, err := strconv.ParseFloat(version2, 32) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
if f1 < f2 { |
||||
return -1 |
||||
} else if f1 > f2 { |
||||
return 1 |
||||
} |
||||
return 0 |
||||
} |
||||
|
||||
func (c *CloudBuilder) Start() (err error) { |
||||
distrosInfo := map[string]DistroInfo{ |
||||
"Ubuntu": DistroInfo{ |
||||
name: "Ubuntu", |
||||
manager: distros.NewUbuntuTemplateManager(c.cfg), |
||||
osDescription: "Other Ubuntu (64-bit)", |
||||
user: "ubuntu", |
||||
}, |
||||
"Fedora": DistroInfo{ |
||||
name: "Fedora", |
||||
manager: distros.NewFedoraTemplateManager(c.cfg), |
||||
osDescription: "Fedora Linux (64 bit)", |
||||
user: "fedora", |
||||
}, |
||||
"AlmaLinux": DistroInfo{ |
||||
name: "AlmaLinux", |
||||
manager: distros.NewAlmaLinuxTemplateManager(c.cfg), |
||||
osDescription: "Other CentOS (64-bit)", |
||||
user: "almalinux", |
||||
}, |
||||
} |
||||
// The elements of this slice must be keys of the map above
|
||||
// TODO use environment variable
|
||||
distrosToCheck := []string{"AlmaLinux"} |
||||
existingVersions := c.getExistingTemplateVersions() |
||||
for _, distro := range distrosToCheck { |
||||
distroInfo := distrosInfo[distro] |
||||
distroManager := distroInfo.manager |
||||
var newVersion, codename string |
||||
newVersion, codename, err = distroManager.GetLatestVersion() |
||||
if err != nil { |
||||
return |
||||
} |
||||
log.Debug().Str("newVersion", newVersion).Str("codename", codename).Msg(distro) |
||||
|
||||
curVersion, ok := existingVersions[distro] |
||||
if ok && c.versionStringCompare(newVersion, curVersion) <= 0 { |
||||
log.Debug(). |
||||
Str("distro", distro). |
||||
Str("version", curVersion). |
||||
Msg("Existing version is up to date, skipping") |
||||
continue |
||||
} |
||||
log.Info().Str("distro", distro).Msg("Existing template is out of date, creating a new one") |
||||
|
||||
var template *cloudstack.Template |
||||
template, err = c.createNewTemplate(newVersion, codename, &distroInfo) |
||||
if err != nil { |
||||
return |
||||
} |
||||
|
||||
vmName := strings.Join([]string{ |
||||
strings.ReplaceAll(strings.ToLower(distro), " ", "-"), |
||||
strings.ReplaceAll(newVersion, ".", "-"), |
||||
"test", |
||||
}, "-") |
||||
var vm *cloudstack.VirtualMachine |
||||
vm, err = c.CreateVM(vmName, template.Id) |
||||
if err != nil { |
||||
return |
||||
} |
||||
|
||||
if err = c.sendEmailNotification(template.Name, vm, distroInfo.user); err != nil { |
||||
return |
||||
} |
||||
} |
||||
return |
||||
} |
||||
@ -0,0 +1,33 @@ |
||||
From: {{ .cfg.EmailSenderName }} <{{ .cfg.EmailSender }}> |
||||
To: {{ .cfg.EmailRecipient }} |
||||
Reply-To: {{ .cfg.EmailReplyTo }} |
||||
Subject: New VM Template: {{ .templateName }} |
||||
Date: {{ .date }} |
||||
|
||||
Hello syscom, |
||||
|
||||
This is an automated message from cloudbuild, the CSC VM template |
||||
builder. |
||||
|
||||
A new VM template, {{ .templateName }}, has been uploaded to CloudStack. |
||||
It is not public. A new VM, {{ .vm.Name }}, has been created from this |
||||
template. You can SSH into this VM from biloba or chamomile by running |
||||
the following: |
||||
|
||||
ssh -i /var/lib/cloudstack/management/.ssh/id_rsa {{ .vmUser }}@{{ (index .vm.Nic 0).IpAddress }} |
||||
|
||||
Please login to the VM and verify that everything is working correctly. |
||||
Once you have done this, please login to CloudStack with the admin account |
||||
and perform the following: |
||||
|
||||
1. Delete the VM (enable the "Expunge" option too). |
||||
2. Make the template public: |
||||
From the web page for the template, click the "Update Template Sharing" |
||||
circular button in the top right corner, and toggle the "Public" |
||||
and "Featured" sliders. |
||||
3. Delete the old template (if it is not being used by any VMs): |
||||
From the web page for the old template, click the "Zones" tab, then |
||||
press the red circular "Delete" button beside "{{ .cfg.CloudstackZoneName }}". |
||||
|
||||
Sincerely, |
||||
cloudbuild |
||||
@ -0,0 +1,399 @@ |
||||
// Package cloudstack provides utilities for interacting with a CloudStack
|
||||
// management server.
|
||||
package cloudstack |
||||
|
||||
/* |
||||
* See https://cloudstack.apache.org/api/apidocs-4.16/ for API docs
|
||||
*/ |
||||
|
||||
import ( |
||||
"bytes" |
||||
"crypto/hmac" |
||||
"crypto/sha1" |
||||
"encoding/base64" |
||||
"encoding/json" |
||||
"errors" |
||||
"fmt" |
||||
"io" |
||||
"net/http" |
||||
"net/url" |
||||
"sort" |
||||
"strings" |
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" |
||||
) |
||||
|
||||
// A CloudstackClient interacts with a CloudStack management server via the
|
||||
// REST API.
|
||||
type CloudstackClient struct { |
||||
cfg *config.Config |
||||
cachedZoneId string |
||||
cachedServiceOfferingId string |
||||
} |
||||
|
||||
// New returns a CloudStack client with the given config values.
|
||||
func New(cfg *config.Config) *CloudstackClient { |
||||
return &CloudstackClient{cfg: cfg} |
||||
} |
||||
|
||||
// CloudStack uses java.net.URLEncoder when calculating signatures:
|
||||
// https://github.com/apache/cloudstack/blob/main/server/src/main/java/com/cloud/api/ApiServer.java
|
||||
// Unfortunately java.net.URLEncoder transforms "~" into "%7E".
|
||||
// This is a bug: https://bugs.openjdk.org/browse/JDK-8204530
|
||||
// So, we need to match Java's behaviour so that the server does not
|
||||
// reject our signature.
|
||||
// (I noticed this problem when passing a `url` parameter to registerTemplate
|
||||
// which contained a tilde.)
|
||||
// TODO: file a GitHub issue in the cloudmonkey repo
|
||||
func urlQueryEscape(value string) string { |
||||
return strings.ReplaceAll(url.QueryEscape(value), "~", "%7E") |
||||
} |
||||
|
||||
// Adapted from https://github.com/apache/cloudstack-cloudmonkey/blob/main/cmd/network.go
|
||||
func encodeRequestParams(params map[string]string) string { |
||||
if params == nil { |
||||
return "" |
||||
} |
||||
|
||||
keys := make([]string, 0, len(params)) |
||||
for key := range params { |
||||
keys = append(keys, key) |
||||
} |
||||
sort.Strings(keys) |
||||
|
||||
var buf bytes.Buffer |
||||
for _, key := range keys { |
||||
value := params[key] |
||||
if buf.Len() > 0 { |
||||
buf.WriteByte('&') |
||||
} |
||||
buf.WriteString(key) |
||||
buf.WriteString("=") |
||||
buf.WriteString(urlQueryEscape(value)) |
||||
} |
||||
return buf.String() |
||||
} |
||||
|
||||
// See http://docs.cloudstack.apache.org/en/4.16.0.0/developersguide/dev.html#how-to-sign-an-api-call-with-python
|
||||
func (client *CloudstackClient) createURL(params map[string]string) string { |
||||
cfg := client.cfg |
||||
params["apiKey"] = cfg.CloudstackApiKey |
||||
params["response"] = "json" |
||||
|
||||
// adapted from https://github.com/apache/cloudstack-cloudmonkey/blob/main/cmd/network.go
|
||||
encodedParams := encodeRequestParams(params) |
||||
mac := hmac.New(sha1.New, []byte(cfg.CloudstackSecretKey)) |
||||
mac.Write([]byte(strings.Replace(strings.ToLower(encodedParams), "+", "%20", -1))) |
||||
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) |
||||
encodedParams += fmt.Sprintf("&signature=%s", urlQueryEscape(signature)) |
||||
|
||||
return fmt.Sprintf("%s?%s", cfg.CloudstackApiBaseUrl, encodedParams) |
||||
} |
||||
|
||||
func getDeserializedResponse(url string, unmarshaledValue interface{}) { |
||||
req, err := http.NewRequest("GET", url, nil) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
resp, err := http.DefaultClient.Do(req) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
defer resp.Body.Close() |
||||
buf, err := io.ReadAll(resp.Body) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
if err = json.Unmarshal(buf, unmarshaledValue); err != nil { |
||||
panic(err) |
||||
} |
||||
} |
||||
|
||||
type ErrorInfo struct { |
||||
ErrorCode int `json:"errorcode"` |
||||
ErrorText string `json:"errortext"` |
||||
} |
||||
|
||||
func checkErrorInfo(data *ErrorInfo) { |
||||
if data.ErrorCode != 0 { |
||||
panic(fmt.Sprintf("Error %d: %s", data.ErrorCode, data.ErrorText)) |
||||
} |
||||
} |
||||
|
||||
type ListDomainsResponse struct { |
||||
ErrorInfo |
||||
Count int `json:"count"` |
||||
Domain []struct { |
||||
Name string `json:"name"` |
||||
Id string `json:"id"` |
||||
} `json:"domain"` |
||||
} |
||||
|
||||
// GetDomainId returns the ID of the domain with the given name.
|
||||
func (client *CloudstackClient) GetDomainId(domainName string) string { |
||||
url := client.createURL(map[string]string{ |
||||
"command": "listDomains", |
||||
"details": "min", |
||||
"name": domainName, |
||||
}) |
||||
responseWrapper := struct { |
||||
Response ListDomainsResponse `json:"listdomainsresponse"` |
||||
}{} |
||||
getDeserializedResponse(url, &responseWrapper) |
||||
data := &responseWrapper.Response |
||||
checkErrorInfo(&data.ErrorInfo) |
||||
if data.Count != 1 { |
||||
panic("there should be one domain found") |
||||
} |
||||
return data.Domain[0].Id |
||||
} |
||||
|
||||
type Template struct { |
||||
Created string `json:"created"` |
||||
Domain string `json:"domain"` |
||||
DomainId string `json:"domainid"` |
||||
Id string `json:"id"` |
||||
Name string `json:"name"` |
||||
IsReady bool `json:"isready"` |
||||
IsFeatured bool `json:"isfeatured"` |
||||
IsPublic bool `json:"ispublic"` |
||||
Status string `json:"status"` |
||||
DownloadDetails []struct { |
||||
DownloadPercent string `json:"downloadPercent"` |
||||
DownloadState string `json:"downloadState"` |
||||
} `json:"downloaddetails"` |
||||
} |
||||
|
||||
type ListTemplatesResponse struct { |
||||
ErrorInfo |
||||
Count int `json:"count"` |
||||
Template []Template `json:"template"` |
||||
} |
||||
|
||||
func (client *CloudstackClient) listTemplates(templateID string) []Template { |
||||
params := map[string]string{ |
||||
"command": "listTemplates", |
||||
"details": "min", |
||||
"templatefilter": "self", |
||||
} |
||||
if templateID != "" { |
||||
params["id"] = templateID |
||||
} |
||||
url := client.createURL(params) |
||||
responseWrapper := struct { |
||||
Response ListTemplatesResponse `json:"listtemplatesresponse"` |
||||
}{} |
||||
getDeserializedResponse(url, &responseWrapper) |
||||
data := &responseWrapper.Response |
||||
checkErrorInfo(&data.ErrorInfo) |
||||
return data.Template |
||||
} |
||||
|
||||
func (client *CloudstackClient) ListTemplates() []Template { |
||||
return client.listTemplates("") |
||||
} |
||||
|
||||
func (client *CloudstackClient) GetTemplate(templateID string) (*Template, error) { |
||||
templates := client.listTemplates(templateID) |
||||
if len(templates) == 0 { |
||||
return nil, errors.New("Template not found") |
||||
} |
||||
return &templates[0], nil |
||||
} |
||||
|
||||
type ListZonesResponse struct { |
||||
ErrorInfo |
||||
Count int `json:"count"` |
||||
Zone []struct { |
||||
Id string `json:"id"` |
||||
Name string `json:"name"` |
||||
} `json:"zone"` |
||||
} |
||||
|
||||
func (client *CloudstackClient) getZoneId() (string, error) { |
||||
if client.cachedZoneId != "" { |
||||
return client.cachedZoneId, nil |
||||
} |
||||
zoneName := client.cfg.CloudstackZoneName |
||||
url := client.createURL(map[string]string{ |
||||
"command": "listZones", |
||||
"name": zoneName, |
||||
}) |
||||
responseWrapper := struct { |
||||
Response ListZonesResponse `json:"listzonesresponse"` |
||||
}{} |
||||
getDeserializedResponse(url, &responseWrapper) |
||||
data := &responseWrapper.Response |
||||
checkErrorInfo(&data.ErrorInfo) |
||||
if data.Count != 1 { |
||||
return "", errors.New(fmt.Sprintf("Expected 1 zone for '%s'; got %d", zoneName, data.Count)) |
||||
} |
||||
zoneId := data.Zone[0].Id |
||||
client.cachedZoneId = zoneId |
||||
return zoneId, nil |
||||
} |
||||
|
||||
type ListServiceOfferingsResponse struct { |
||||
ErrorInfo |
||||
Count int `json:"count"` |
||||
ServiceOffering []struct { |
||||
Id string `json:"id"` |
||||
Name string `json:"name"` |
||||
} `json:"serviceoffering"` |
||||
} |
||||
|
||||
func (client *CloudstackClient) getServiceOfferingId() (string, error) { |
||||
if client.cachedServiceOfferingId != "" { |
||||
return client.cachedServiceOfferingId, nil |
||||
} |
||||
serviceOfferingName := client.cfg.CloudstackServiceOfferingName |
||||
url := client.createURL(map[string]string{ |
||||
"command": "listServiceOfferings", |
||||
"name": serviceOfferingName, |
||||
}) |
||||
responseWrapper := struct { |
||||
Response ListServiceOfferingsResponse `json:"listserviceofferingsresponse"` |
||||
}{} |
||||
getDeserializedResponse(url, &responseWrapper) |
||||
data := &responseWrapper.Response |
||||
checkErrorInfo(&data.ErrorInfo) |
||||
if data.Count != 1 { |
||||
return "", errors.New(fmt.Sprintf("Expected 1 service offering for '%s'; got %d", serviceOfferingName, data.Count)) |
||||
} |
||||
serviceOfferingId := data.ServiceOffering[0].Id |
||||
client.cachedServiceOfferingId = serviceOfferingId |
||||
return serviceOfferingId, nil |
||||
} |
||||
|
||||
// Note: if an OS type you need isn't available in CloudStack, you can
|
||||
// create a new one with addGuestOs
|
||||
|
||||
type OsType struct { |
||||
Id string `json:"id"` |
||||
Description string `json:"description"` |
||||
} |
||||
|
||||
type ListOsTypesResponse struct { |
||||
ErrorInfo |
||||
Count int `json:"count"` |
||||
OsType []OsType `json:"ostype"` |
||||
} |
||||
|
||||
func (client *CloudstackClient) GetOsTypeId(osDescription string) (string, error) { |
||||
url := client.createURL(map[string]string{ |
||||
"command": "listOsTypes", |
||||
"description": osDescription, |
||||
}) |
||||
responseWrapper := struct { |
||||
Response ListOsTypesResponse `json:"listostypesresponse"` |
||||
}{} |
||||
getDeserializedResponse(url, &responseWrapper) |
||||
data := &responseWrapper.Response |
||||
checkErrorInfo(&data.ErrorInfo) |
||||
if data.Count != 1 { |
||||
return "", errors.New(fmt.Sprintf("Expected 1 OS type for '%s'; got %d", osDescription, data.Count)) |
||||
} |
||||
return data.OsType[0].Id, nil |
||||
} |
||||
|
||||
type RegisterTemplateResponse struct { |
||||
ErrorInfo |
||||
Count int `json:"count"` |
||||
Template []Template `json:"template"` |
||||
} |
||||
|
||||
func (client *CloudstackClient) RegisterTemplate(name string, downloadUrl string, osTypeId string) (*Template, error) { |
||||
zoneId, err := client.getZoneId() |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
url := client.createURL(map[string]string{ |
||||
"command": "registerTemplate", |
||||
"displaytext": name, |
||||
"format": "QCOW2", |
||||
"hypervisor": "KVM", |
||||
"name": name, |
||||
"url": downloadUrl, |
||||
"isextractable": "true", |
||||
"isfeatured": "false", |
||||
"ispublic": "false", |
||||
"ostypeid": osTypeId, |
||||
"requireshvm": "true", |
||||
"zoneid": zoneId, |
||||
}) |
||||
responseWrapper := struct { |
||||
Response RegisterTemplateResponse `json:"registertemplateresponse"` |
||||
}{} |
||||
getDeserializedResponse(url, &responseWrapper) |
||||
data := &responseWrapper.Response |
||||
checkErrorInfo(&data.ErrorInfo) |
||||
return &data.Template[0], nil |
||||
} |
||||
|
||||
type VirtualMachine struct { |
||||
Id string `json:"id"` |
||||
DisplayName string `json:"displayname"` |
||||
Name string `json:"name"` |
||||
State string `json:"state"` |
||||
Nic []struct { |
||||
Id string `json:"id"` |
||||
IpAddress string `json:"ipaddress"` |
||||
IsDefault bool `json:"isdefault"` |
||||
} `json:"nic"` |
||||
} |
||||
|
||||
type DeployVirtualMachineResponse struct { |
||||
ErrorInfo |
||||
Id string `json:"id"` |
||||
JobId string `json:"jobid"` |
||||
} |
||||
|
||||
func (client *CloudstackClient) DeployVirtualMachine( |
||||
name string, templateID string, |
||||
) (vmID string, err error) { |
||||
serviceOfferingId, err := client.getServiceOfferingId() |
||||
if err != nil { |
||||
return |
||||
} |
||||
zoneId, err := client.getZoneId() |
||||
if err != nil { |
||||
return |
||||
} |
||||
url := client.createURL(map[string]string{ |
||||
"command": "deployVirtualMachine", |
||||
"serviceofferingid": serviceOfferingId, |
||||
"templateid": templateID, |
||||
"zoneid": zoneId, |
||||
"name": name, |
||||
"displayname": name, |
||||
"keypair": client.cfg.CloudstackKeypairName, |
||||
}) |
||||
responseWrapper := struct { |
||||
Response DeployVirtualMachineResponse `json:"deployvirtualmachineresponse"` |
||||
}{} |
||||
getDeserializedResponse(url, &responseWrapper) |
||||
data := &responseWrapper.Response |
||||
checkErrorInfo(&data.ErrorInfo) |
||||
return data.Id, nil |
||||
} |
||||
|
||||
type ListVirtualMachinesResponse struct { |
||||
ErrorInfo |
||||
Count int `json:"count"` |
||||
VirtualMachine []VirtualMachine `json:"virtualmachine"` |
||||
} |
||||
|
||||
func (client *CloudstackClient) GetVirtualMachine(vmID string) (virtualMachine *VirtualMachine, err error) { |
||||
url := client.createURL(map[string]string{ |
||||
"command": "listVirtualMachines", |
||||
"id": vmID, |
||||
}) |
||||
responseWrapper := struct { |
||||
Response ListVirtualMachinesResponse `json:"listvirtualmachinesresponse"` |
||||
}{} |
||||
getDeserializedResponse(url, &responseWrapper) |
||||
data := &responseWrapper.Response |
||||
checkErrorInfo(&data.ErrorInfo) |
||||
virtualMachine = &data.VirtualMachine[0] |
||||
return |
||||
} |
||||
@ -0,0 +1,62 @@ |
||||
// Package config provides utilities for reading and storing configuration
|
||||
// values.
|
||||
package config |
||||
|
||||
import ( |
||||
"os" |
||||
) |
||||
|
||||
// A Config holds all of the configuration values needed for the program.
|
||||
type Config struct { |
||||
CloudstackApiKey string |
||||
CloudstackSecretKey string |
||||
CloudstackApiBaseUrl string |
||||
CloudstackZoneName string |
||||
CloudstackServiceOfferingName string |
||||
CloudstackKeypairName string |
||||
UploadDirectory string |
||||
UploadBaseUrl string |
||||
MirrorHost string |
||||
EmailServer string |
||||
EmailSender string |
||||
EmailSenderName string |
||||
EmailRecipient string |
||||
EmailReplyTo string |
||||
} |
||||
|
||||
// New returns a Config filled with values read from environment variables.
|
||||
// It panics if a required environment variable is empty or not set.
|
||||
func New() *Config { |
||||
cfg := &Config{ |
||||
CloudstackApiKey: os.Getenv("CLOUDSTACK_API_KEY"), |
||||
CloudstackSecretKey: os.Getenv("CLOUDSTACK_SECRET_KEY"), |
||||
UploadDirectory: os.Getenv("UPLOAD_DIRECTORY"), |
||||
UploadBaseUrl: os.Getenv("UPLOAD_BASE_URL"), |
||||
} |
||||
if cfg.CloudstackApiKey == "" { |
||||
panic("CLOUDSTACK_API_KEY is empty or not set") |
||||
} |
||||
if cfg.CloudstackSecretKey == "" { |
||||
panic("CLOUDSTACK_SECRET_KEY is empty or not set") |
||||
} |
||||
if cfg.UploadDirectory == "" { |
||||
panic("UPLOAD_DIRECTORY is empty or not set") |
||||
} |
||||
if cfg.UploadBaseUrl == "" { |
||||
panic("UPLOAD_BASE_URL is empty or not set") |
||||
} |
||||
|
||||
// These should never change
|
||||
cfg.CloudstackApiBaseUrl = "https://cloud.csclub.uwaterloo.ca/client/api" |
||||
cfg.CloudstackZoneName = "Zone1" |
||||
cfg.CloudstackServiceOfferingName = "Small Instance" |
||||
cfg.CloudstackKeypairName = "management-keypair" |
||||
cfg.MirrorHost = "mirror.csclub.uwaterloo.ca" |
||||
cfg.EmailServer = "mail.csclub.uwaterloo.ca:25" |
||||
cfg.EmailSender = "cloudbuild@csclub.uwaterloo.ca" |
||||
cfg.EmailSenderName = "cloudbuild" |
||||
// TODO: change recipient to syscom
|
||||
cfg.EmailRecipient = "merenber@csclub.uwaterloo.ca" |
||||
cfg.EmailReplyTo = "no-reply@csclub.uwaterloo.ca" |
||||
return cfg |
||||
} |
||||
@ -0,0 +1,139 @@ |
||||
package distros |
||||
|
||||
import ( |
||||
"errors" |
||||
"fmt" |
||||
"math" |
||||
"net/http" |
||||
"regexp" |
||||
"strconv" |
||||
|
||||
"github.com/rs/zerolog/log" |
||||
"golang.org/x/net/html" |
||||
"libguestfs.org/guestfs" |
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" |
||||
) |
||||
|
||||
// TODO: reduce code duplication with FedoraTemplateManager
|
||||
|
||||
type AlmaLinuxTemplateManager struct { |
||||
TemplateManager |
||||
} |
||||
|
||||
// NewAlmaLinuxTemplateManager returns a AlmaLinuxTemplateManager with the given
|
||||
// config values.
|
||||
func NewAlmaLinuxTemplateManager(cfg *config.Config) *AlmaLinuxTemplateManager { |
||||
logger := log.With().Str("distro", "almalinux").Logger() |
||||
almaLinuxTemplateManager := AlmaLinuxTemplateManager{ |
||||
TemplateManager{ |
||||
cfg: cfg, |
||||
logger: &logger, |
||||
impl: nil, |
||||
}, |
||||
} |
||||
almaLinuxTemplateManager.TemplateManager.impl = &almaLinuxTemplateManager |
||||
return &almaLinuxTemplateManager |
||||
} |
||||
|
||||
var floatNumberSlashPattern *regexp.Regexp = regexp.MustCompile("^\\d+(\\.\\d)?/$") |
||||
|
||||
func getMaxFloatVersionFromHtml(node *html.Node, maxVersion *float64) { |
||||
if isHyperlink(node) { |
||||
text := node.FirstChild.Data |
||||
if numberSlashPattern.FindString(text) != "" { |
||||
version, _ := strconv.ParseFloat(text[:len(text)-1], 64) |
||||
if version > *maxVersion { |
||||
*maxVersion = version |
||||
} |
||||
} |
||||
return |
||||
} |
||||
for c := node.FirstChild; c != nil; c = c.NextSibling { |
||||
getMaxFloatVersionFromHtml(c, maxVersion) |
||||
} |
||||
} |
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) GetLatestVersion() (version string, codename string, err error) { |
||||
resp, err := http.Get("https://mirror.csclub.uwaterloo.ca/almalinux/") |
||||
if err != nil { |
||||
return |
||||
} |
||||
defer resp.Body.Close() |
||||
node, err := html.Parse(resp.Body) |
||||
if err != nil { |
||||
return |
||||
} |
||||
var floatVersion float64 |
||||
getMaxFloatVersionFromHtml(node, &floatVersion) |
||||
if floatVersion == 0 { |
||||
err = errors.New("Could not determine latest AlmaLinux version from HTML") |
||||
} else { |
||||
version = strconv.FormatFloat(floatVersion, 'f', -1, 64) |
||||
// AlmaLinux doesn't have codenames, only numbered versions
|
||||
codename = version |
||||
} |
||||
return |
||||
} |
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) DownloadTemplate(codename string) (path string, err error) { |
||||
// TODO: REMOVE THIS
|
||||
//if true {
|
||||
// return "AlmaLinux-9-GenericCloud-latest.x86_64.qcow2", nil
|
||||
//}
|
||||
version := codename |
||||
floatVersion, err := strconv.ParseFloat(version, 64) |
||||
if err != nil { |
||||
return |
||||
} |
||||
majorVersion := int(math.Floor(floatVersion)) |
||||
filename := fmt.Sprintf("AlmaLinux-%d-GenericCloud-latest.x86_64.qcow2", majorVersion) |
||||
url := fmt.Sprintf("https://mirror.csclub.uwaterloo.ca/almalinux/%s/cloud/x86_64/images/%s", version, filename) |
||||
return mgr.DownloadTemplateGeneric(filename, url) |
||||
} |
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) GetIpv6ScriptsTemplateData() map[string]string { |
||||
return map[string]string{ |
||||
"networkService": "NetworkManager.service", |
||||
"networkTarget": "network-online.target", |
||||
} |
||||
} |
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) addCloudInitSnippet(handle *guestfs.Guestfs) error { |
||||
path := "/etc/cloud/cloud.cfg.d/99_csclub.cfg" |
||||
mgr.logger.Debug().Msg("Writing to " + path) |
||||
return handle.Write(path, getResource("fedora-cloud-init")) |
||||
} |
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) HasSELinuxEnabled() bool { |
||||
return true |
||||
} |
||||
|
||||
var almaLinuxYumRepoBaseUrlPattern *regexp.Regexp = regexp.MustCompile( |
||||
"^(?P<scheme>https?://)[A-Za-z0-9./-]+(?P<path>/almalinux/\\$releasever/[A-Za-z0-9./$-]+)$", |
||||
) |
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) transformAlmaLinuxYumRepoBaseUrl(url string) string { |
||||
submatches := almaLinuxYumRepoBaseUrlPattern.FindStringSubmatch(url) |
||||
if submatches != nil { |
||||
scheme, path := submatches[1], submatches[2] |
||||
url = scheme + mgr.cfg.MirrorHost + path |
||||
} |
||||
return url |
||||
} |
||||
|
||||
func (mgr *AlmaLinuxTemplateManager) PerformDistroSpecificModifications(handle *guestfs.Guestfs) (err error) { |
||||
if err = mgr.setChronyOptions(handle, "/etc/chrony.conf"); err != nil { |
||||
return |
||||
} |
||||
if err = mgr.setNetworkManagerOptions(handle); err != nil { |
||||
return |
||||
} |
||||
if err = mgr.addCloudInitSnippet(handle); err != nil { |
||||
return |
||||
} |
||||
if err = mgr.replaceYumRepoMirrorUrls(handle, mgr.transformAlmaLinuxYumRepoBaseUrl); err != nil { |
||||
return |
||||
} |
||||
return |
||||
} |
||||
@ -0,0 +1,161 @@ |
||||
package distros |
||||
|
||||
import ( |
||||
"errors" |
||||
"fmt" |
||||
"net/http" |
||||
"regexp" |
||||
"strconv" |
||||
|
||||
"github.com/rs/zerolog/log" |
||||
"golang.org/x/net/html" |
||||
"libguestfs.org/guestfs" |
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" |
||||
) |
||||
|
||||
type FedoraTemplateManager struct { |
||||
TemplateManager |
||||
} |
||||
|
||||
// NewFedoraTemplateManager returns a FedoraTemplateManager with the given
|
||||
// config values.
|
||||
func NewFedoraTemplateManager(cfg *config.Config) *FedoraTemplateManager { |
||||
logger := log.With().Str("distro", "fedora").Logger() |
||||
fedoraTemplateManager := FedoraTemplateManager{ |
||||
TemplateManager{ |
||||
cfg: cfg, |
||||
logger: &logger, |
||||
impl: nil, |
||||
}, |
||||
} |
||||
fedoraTemplateManager.TemplateManager.impl = &fedoraTemplateManager |
||||
return &fedoraTemplateManager |
||||
} |
||||
|
||||
var numberSlashPattern *regexp.Regexp = regexp.MustCompile("^\\d+/$") |
||||
|
||||
func isHyperlink(node *html.Node) bool { |
||||
return node.Type == html.ElementNode && node.Data == "a" && |
||||
node.FirstChild != nil && node.FirstChild.Type == html.TextNode |
||||
} |
||||
|
||||
func getMatchingLinkFromHtml(node *html.Node, pattern *regexp.Regexp) string { |
||||
if isHyperlink(node) { |
||||
text := node.FirstChild.Data |
||||
return pattern.FindString(text) |
||||
} |
||||
for c := node.FirstChild; c != nil; c = c.NextSibling { |
||||
result := getMatchingLinkFromHtml(c, pattern) |
||||
if result != "" { |
||||
return result |
||||
} |
||||
} |
||||
return "" |
||||
} |
||||
|
||||
func getMaxVersionFromHtml(node *html.Node, maxVersion *int64) { |
||||
if isHyperlink(node) { |
||||
text := node.FirstChild.Data |
||||
if numberSlashPattern.FindString(text) != "" { |
||||
version, _ := strconv.ParseInt(text[:len(text)-1], 10, 64) |
||||
if version > *maxVersion { |
||||
*maxVersion = version |
||||
} |
||||
} |
||||
return |
||||
} |
||||
for c := node.FirstChild; c != nil; c = c.NextSibling { |
||||
getMaxVersionFromHtml(c, maxVersion) |
||||
} |
||||
} |
||||
|
||||
func (mgr *FedoraTemplateManager) GetLatestVersion() (version string, codename string, err error) { |
||||
resp, err := http.Get("https://mirror.csclub.uwaterloo.ca/fedora/linux/releases/") |
||||
if err != nil { |
||||
return |
||||
} |
||||
defer resp.Body.Close() |
||||
node, err := html.Parse(resp.Body) |
||||
if err != nil { |
||||
return |
||||
} |
||||
var intVersion int64 |
||||
getMaxVersionFromHtml(node, &intVersion) |
||||
if intVersion == 0 { |
||||
err = errors.New("Could not determine latest Fedora version from HTML") |
||||
} else { |
||||
version = strconv.FormatInt(intVersion, 10) |
||||
// Fedora doesn't have codenames, only numbered versions
|
||||
codename = version |
||||
} |
||||
return |
||||
} |
||||
|
||||
func (mgr *FedoraTemplateManager) DownloadTemplate(codename string) (path string, err error) { |
||||
version := codename |
||||
pattern := regexp.MustCompile(fmt.Sprintf("^Fedora-Cloud-Base-%s-\\d+(\\.\\d+)?\\.x86_64\\.qcow2$", version)) |
||||
imagesUrl := fmt.Sprintf("https://mirror.csclub.uwaterloo.ca/fedora/linux/releases/%s/Cloud/x86_64/images/", version) |
||||
resp, err := http.Get(imagesUrl) |
||||
if err != nil { |
||||
return |
||||
} |
||||
defer resp.Body.Close() |
||||
node, err := html.Parse(resp.Body) |
||||
if err != nil { |
||||
return |
||||
} |
||||
filename := getMatchingLinkFromHtml(node, pattern) |
||||
if filename == "" { |
||||
err = errors.New("Could not find hyperlink for Fedora Cloud image") |
||||
return |
||||
} |
||||
url := imagesUrl + filename |
||||
return mgr.DownloadTemplateGeneric(filename, url) |
||||
} |
||||
|
||||
func (mgr *FedoraTemplateManager) GetIpv6ScriptsTemplateData() map[string]string { |
||||
return map[string]string{ |
||||
"networkService": "NetworkManager.service", |
||||
"networkTarget": "network-online.target", |
||||
} |
||||
} |
||||
|
||||
func (mgr *FedoraTemplateManager) addCloudInitSnippet(handle *guestfs.Guestfs) error { |
||||
path := "/etc/cloud/cloud.cfg.d/99_csclub.cfg" |
||||
mgr.logger.Debug().Msg("Writing to " + path) |
||||
return handle.Write(path, getResource("fedora-cloud-init")) |
||||
} |
||||
|
||||
func (mgr *FedoraTemplateManager) HasSELinuxEnabled() bool { |
||||
return true |
||||
} |
||||
|
||||
var fedoraYumRepoBaseUrlPattern *regexp.Regexp = regexp.MustCompile( |
||||
"^(?P<scheme>https?://)[A-Za-z0-9./-]+(?P<path>/fedora/linux/[A-Za-z0-9./$-]+)$", |
||||
) |
||||
|
||||
func (mgr *FedoraTemplateManager) transformFedoraYumRepoBaseUrl(url string) string { |
||||
submatches := fedoraYumRepoBaseUrlPattern.FindStringSubmatch(url) |
||||
if submatches != nil { |
||||
scheme, path := submatches[1], submatches[2] |
||||
url = scheme + mgr.cfg.MirrorHost + path |
||||
} |
||||
return url |
||||
} |
||||
|
||||
func (mgr *FedoraTemplateManager) PerformDistroSpecificModifications(handle *guestfs.Guestfs) (err error) { |
||||
if err = mgr.setChronyOptions(handle, "/etc/chrony.conf"); err != nil { |
||||
return |
||||
} |
||||
if err = mgr.setNetworkManagerOptions(handle); err != nil { |
||||
return |
||||
} |
||||
if err = mgr.addCloudInitSnippet(handle); err != nil { |
||||
return |
||||
} |
||||
if err = mgr.replaceYumRepoMirrorUrls(handle, mgr.transformFedoraYumRepoBaseUrl); err != nil { |
||||
return |
||||
} |
||||
return |
||||
} |
||||
@ -0,0 +1,22 @@ |
||||
#!/bin/bash |
||||
|
||||
set -ex |
||||
|
||||
nmcli -f name -terse c | grep "^cloud-init e" | \ |
||||
while read conn_name; do |
||||
iface=$(echo "$conn_name" | cut -d ' ' -f 2) |
||||
nmcli c modify "$conn_name" ipv6.method link-local |
||||
nmcli d reapply $iface |
||||
done |
||||
|
||||
ip -6 -brief addr show dynamic | \ |
||||
while read iface _ addrs; do |
||||
for addr in $addrs; do |
||||
ip addr del dev $iface $addr |
||||
done |
||||
done |
||||
|
||||
ip -6 route show proto ra | \ |
||||
while read line; do |
||||
ip route del $line |
||||
done |
||||
@ -0,0 +1,20 @@ |
||||
#!/bin/bash |
||||
set -ex |
||||
INTERFACE= |
||||
IPV4_ADDRESS= |
||||
while read interface _ address; do |
||||
if ! echo $address | grep -q '^172\.19\.134\.'; then |
||||
continue |
||||
fi |
||||
INTERFACE=$interface |
||||
IPV4_ADDRESS=$address |
||||
break |
||||
done < <(ip -4 -brief addr show) |
||||
if [ -z "$INTERFACE" ]; then |
||||
echo "Could not find primary interface" >&2 |
||||
exit 1 |
||||
fi |
||||
NUM=$(echo $IPV4_ADDRESS | grep -oP '^172\.19\.134\.\K(\d+)') |
||||
IPV6_ADDRESS="2620:101:f000:4903::$NUM/64" |
||||
ip -6 addr add $IPV6_ADDRESS dev $INTERFACE |
||||
ip -6 route add default via 2620:101:f000:4903::1 dev $INTERFACE |
||||
@ -0,0 +1,8 @@ |
||||
#server ntp.csclub.uwaterloo.ca |
||||
server 129.97.167.12 |
||||
#server ntp.student.cs.uwaterloo.ca |
||||
server 129.97.167.4 |
||||
#server ntp.cs.uwaterloo.ca |
||||
server 129.97.15.14 |
||||
#server ntp.cscf.uwaterloo.ca |
||||
server 129.97.15.15 |
||||
@ -0,0 +1 @@ |
||||
manage_etc_hosts: true |
||||
@ -0,0 +1,9 @@ |
||||
#!/bin/bash |
||||
set -ex |
||||
while read interface _; do |
||||
if [ $interface = lo ]; then |
||||
continue |
||||
fi |
||||
sysctl net.ipv6.conf.$interface.accept_ra=0 |
||||
sysctl net.ipv6.conf.$interface.accept_dad=0 |
||||
done < <(ip -brief link show) |
||||
@ -0,0 +1,12 @@ |
||||
|
||||
UNIVERSITY OF WATERLOO COMPUTER SCIENCE CLUB |
||||
Welcome to the CSC cloud! :) |
||||
This service was made possible thanks to donations from the Mathematics |
||||
Endowment Fund (MEF) and the Computer Science Computing Facility (CSCF). |
||||
|
||||
Documentation: |
||||
https://docs.cloud.csclub.uwaterloo.ca |
||||
Machine usage agreement: |
||||
https://csclub.uwaterloo.ca/resources/machine-usage-agreement |
||||
Questions/concerns can be emailed to: |
||||
syscom@csclub.uwaterloo.ca |
||||
@ -0,0 +1,3 @@ |
||||
[main] |
||||
dhcp = dhclient |
||||
dns = none |
||||
@ -0,0 +1,12 @@ |
||||
search csclub.uwaterloo.ca uwaterloo.ca |
||||
options rotate timeout:1 attempts:1 ndots:2 |
||||
|
||||
# CSC Nameservers |
||||
nameserver 2620:101:f000:4901:c5c::4 |
||||
nameserver 2620:101:f000:7300:c5c::20 |
||||
nameserver 129.97.134.4 |
||||
nameserver 129.97.18.20 |
||||
|
||||
# IST Anycast (fallback) |
||||
#nameserver 129.97.2.1 |
||||
#nameserver 129.97.2.2) |
||||
@ -0,0 +1,2 @@ |
||||
net.ipv6.conf.all.accept_ra = 0 |
||||
net.ipv6.conf.default.accept_ra = 0 |
||||
@ -0,0 +1,5 @@ |
||||
[Time] |
||||
# ntp.csclub.uwaterloo.ca ntp.student.cs.uwaterloo.ca |
||||
NTP=129.97.167.12 129.97.167.4 |
||||
# ntp.cs.uwaterloo.ca ntp.cscf.uwaterloo.ca |
||||
FallbackNTP=129.97.15.14 129.97.15.15 |
||||
@ -0,0 +1,11 @@ |
||||
network: |
||||
version: 2 |
||||
ethernets: |
||||
id0: |
||||
match: |
||||
name: e* |
||||
dhcp4: true |
||||
accept-ra: false |
||||
|
||||
apt_preserve_sources_list: true |
||||
manage_etc_hosts: true |
||||
@ -0,0 +1,526 @@ |
||||
package distros |
||||
|
||||
import ( |
||||
"embed" |
||||
"errors" |
||||
"fmt" |
||||
"io" |
||||
"net/http" |
||||
"net/url" |
||||
"os" |
||||
"regexp" |
||||
"strings" |
||||
"text/template" |
||||
|
||||
"github.com/rs/zerolog" |
||||
"libguestfs.org/guestfs" |
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" |
||||
) |
||||
|
||||
//go:embed resources
|
||||
var res embed.FS |
||||
|
||||
func getResource(filename string) []byte { |
||||
data, err := res.ReadFile("resources/" + filename) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
return data |
||||
} |
||||
|
||||
func getTemplateResource(filename string) *template.Template { |
||||
tmpl, err := template.ParseFS(res, "resources/"+filename) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
return tmpl |
||||
} |
||||
|
||||
// A TemplateManager downloads and modifies VM templates for a distro.
|
||||
type TemplateManager struct { |
||||
cfg *config.Config |
||||
logger *zerolog.Logger |
||||
impl IDistroSpecificTemplateManager |
||||
} |
||||
|
||||
type ITemplateManager interface { |
||||
// GetLatestVersion returns the version number and codename of the
|
||||
// latest version of particular OS (e.g. version="22.04", codename="jammy")
|
||||
GetLatestVersion() (version string, codename string, err error) |
||||
// DownloadTemplate downloads a VM template for a given OS codename
|
||||
// and returns the path to where it was downloaded
|
||||
DownloadTemplate(codename string) (path string, err error) |
||||
// ModifyTemplate makes custom modifications to the downloaded VM template
|
||||
ModifyTemplate(filename string) error |
||||
} |
||||
|
||||
// An IDistroSpecificTemplateManager performs the distro-specific tasks
|
||||
// when modifying a VM template. It is used by the generic TemplateManager.
|
||||
type IDistroSpecificTemplateManager interface { |
||||
// GetIpv6ScriptsTemplateData returns template data used by the
|
||||
// IPv6 scripts. The required keys are "networkService" and
|
||||
// "networkTarget"; the values should be a systemd service and target,
|
||||
// respectively.
|
||||
GetIpv6ScriptsTemplateData() map[string]string |
||||
// PerformDistroSpecificModifications is called after
|
||||
// performDistroAgnosticModifications to modify a template in a
|
||||
// distro-specific way.
|
||||
PerformDistroSpecificModifications(handle *guestfs.Guestfs) error |
||||
// HasSELinuxEnabled returns whether SELinux is enabled for this distro
|
||||
HasSELinuxEnabled() bool |
||||
} |
||||
|
||||
func (mgr *TemplateManager) DownloadTemplateGeneric(filename, url string) (path string, err error) { |
||||
mgr.logger.Debug().Str("url", url).Msg("Downloading template") |
||||
resp, err := http.Get(url) |
||||
if err != nil { |
||||
return |
||||
} |
||||
defer resp.Body.Close() |
||||
// TODO: download to somewhere in /tmp
|
||||
path = filename |
||||
out, err := os.Create(path) |
||||
if err != nil { |
||||
return |
||||
} |
||||
defer out.Close() |
||||
_, err = io.Copy(out, resp.Body) |
||||