commit 08f73711988200e8dbcc7a3e8c49e482a7f16bc9 Author: Max Erenberg Date: Tue Jun 21 02:36:42 2022 -0400 first commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ebfc496 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/cloudbuild +/guestfs +*.swp diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..33740f7 --- /dev/null +++ b/Makefile @@ -0,0 +1,40 @@ +DEPS_DIR = guestfs/deps +LIBRARY_PATH = $(DEPS_DIR)/usr/lib/x86_64-linux-gnu:$(DEPS_DIR)/lib/x86_64-linux-gnu +LIBGUESTFS_PATH = guestfs/appliance +LIBGUESTFS_HV = scripts/qemu.sh +APPLIANCE_VERSION = 1.46.0 +DOWNLOAD_DEPS_ARGS = + +# Export LIBGUESTFS_DEBUG=1 to debug + +ifeq ($(GUESTFISH),1) + DOWNLOAD_DEPS_ARGS = guestfish +endif + +all: + LIBRARY_PATH=$(LIBRARY_PATH) CGO_LDFLAGS='-l:libvirt.so.0 -l:libyajl.so.2' go build + +run: + LD_LIBRARY_PATH=$(LIBRARY_PATH) LIBGUESTFS_PATH=$(LIBGUESTFS_PATH) LIBGUESTFS_HV=$(LIBGUESTFS_HV) LIBGUESTFS_BACKEND_SETTINGS=force_tcg ./cloudbuild + +guestfish: + LD_LIBRARY_PATH=$(LIBRARY_PATH) LIBGUESTFS_PATH=$(LIBGUESTFS_PATH) LIBGUESTFS_HV=$(LIBGUESTFS_HV) LIBGUESTFS_BACKEND_SETTINGS=force_tcg PATH=guestfs/qemu-utils-deps/usr/bin:$(PATH) $(DEPS_DIR)/usr/bin/guestfish + +deps: + scripts/download-deps.sh $(DOWNLOAD_DEPS_ARGS) + +appliance-download: + cd guestfs && \ + wget https://download.libguestfs.org/binaries/appliance/appliance-$(APPLIANCE_VERSION).tar.xz && \ + tar Jxvf appliance-$(APPLIANCE_VERSION).tar.xz && \ + rm appliance-$(APPLIANCE_VERSION).tar.xz + +appliance: $(DEPS_DIR)usr/bin/supermin + mkdir -p /var/tmp/.guestfs-`id -u` + $(DEPS_DIR)usr/bin/supermin --build --verbose --if-newer --lock /var/tmp/.guestfs-`id -u`/lock --copy-kernel -f ext2 --host-cpu x86_64 $(DEPS_DIR)usr/lib/x86_64-linux-gnu/guestfs/supermin.d -o /var/tmp/.guestfs-`id -u`/appliance.d + mv /var/tmp/.guestfs-`id -u`/appliance.d guestfs/appliance + +$(DEPS_DIR)usr/bin/supermin: + cd guestfs && apt download supermin && dpkg -x supermin_*.deb deps && rm supermin_*.deb + +.PHONY: all run guestfish deps appliance-download appliance diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f0b8000 --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module git.csclub.uwaterloo.ca/cloud/cloudbuild + +go 1.17 + +replace libguestfs.org/guestfs => ./guestfs + +require ( + github.com/rs/zerolog v1.26.1 + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d + libguestfs.org/guestfs v0.0.0-00010101000000-000000000000 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..00edcf3 --- /dev/null +++ b/go.sum @@ -0,0 +1,33 @@ +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= +github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go new file mode 100644 index 0000000..046087c --- /dev/null +++ b/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "os" + "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + + "git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/cloudbuilder" + "git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" +) + +func setupLogging() { + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}) +} + +func main() { + setupLogging() + cfg := config.New() + builder := cloudbuilder.New(cfg) + + if err := builder.Start(); err != nil { + panic(err) + } +} diff --git a/pkg/cloudbuilder/cloudbuilder.go b/pkg/cloudbuilder/cloudbuilder.go new file mode 100644 index 0000000..dc7e4e2 --- /dev/null +++ b/pkg/cloudbuilder/cloudbuilder.go @@ -0,0 +1,287 @@ +package cloudbuilder + +import ( + "bytes" + _ "embed" + "errors" + "fmt" + "net/smtp" + "os" + "path" + "regexp" + "strconv" + "strings" + "text/template" + "time" + + "github.com/rs/zerolog/log" + + "git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/cloudstack" + "git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" + "git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/distros" +) + +//go:embed email-message.txt +var emailMessageTemplate string + +type CloudBuilder struct { + cfg *config.Config + client *cloudstack.CloudstackClient +} + +func New(cfg *config.Config) *CloudBuilder { + return &CloudBuilder{ + cfg: cfg, + client: cloudstack.New(cfg), + } +} + +func (c *CloudBuilder) moveTemplateToUploadArea(templatePath string) (newPath string, err error) { + newPath = path.Join(c.cfg.UploadDirectory, path.Base(templatePath)) + log.Debug().Msg(fmt.Sprintf("Moving %s to %s", templatePath, newPath)) + err = os.Rename(templatePath, newPath) + return +} + +func (c *CloudBuilder) UploadTemplate( + templatePath string, templateName string, osDescription string, +) (template *cloudstack.Template, err error) { + templatePath, err = c.moveTemplateToUploadArea(templatePath) + if err != nil { + return + } + osTypeId, err := c.client.GetOsTypeId(osDescription) + if err != nil { + return + } + uploadUrl := strings.TrimRight(c.cfg.UploadBaseUrl, "/") + "/" + path.Base(templatePath) + log.Debug().Msg(fmt.Sprintf("Uploading new template from %s", uploadUrl)) + template, err = c.client.RegisterTemplate(templateName, uploadUrl, osTypeId) + if err != nil { + return + } + const pollInterval = 5 * time.Second + // Due to clock drift, we may end up waiting a bit longer than this + const maxTimeToWait = 5 * time.Minute + var timeWaited time.Duration = 0 + for !template.IsReady { + // TODO: make sure that DownloadDetails[0] exists + log.Debug(). + Str("templateName", template.Name). + Str("status", template.Status). + Str("downloadPercent", template.DownloadDetails[0].DownloadPercent). + Str("downloadState", template.DownloadDetails[0].DownloadState). + Msg("Template is not ready yet...") + time.Sleep(pollInterval) + timeWaited += pollInterval + if timeWaited >= maxTimeToWait { + err = errors.New("Timed out waiting for template to be ready") + return + } + template, err = c.client.GetTemplate(template.Id) + if err != nil { + return + } + } + log.Debug().Msg("Deleting " + templatePath) + err = os.Remove(templatePath) + return +} + +func (c *CloudBuilder) CreateVM( + name string, templateID string, +) (virtualMachine *cloudstack.VirtualMachine, err error) { + vmID, err := c.client.DeployVirtualMachine(name, templateID) + if err != nil { + return + } + virtualMachine, err = c.client.GetVirtualMachine(vmID) + if err != nil { + return + } + const pollInterval = 10 * time.Second + // Due to clock drift, we may end up waiting a bit longer than this + const maxTimeToWait = 30 * time.Minute + var timeWaited time.Duration = 0 + log.Debug().Str("templateID", templateID).Msg("Creating new VM") + for virtualMachine.State != "Running" { + log.Debug(). + Str("name", name). + Str("id", virtualMachine.Id). + Str("state", virtualMachine.State). + Msg("VM is not ready yet...") + time.Sleep(pollInterval) + timeWaited += pollInterval + if timeWaited >= maxTimeToWait { + err = errors.New("Timed out waiting for VM to be ready") + return + } + virtualMachine, err = c.client.GetVirtualMachine(vmID) + if err != nil { + return + } + } + log.Debug().Msg("VM successfully created") + return +} + +type DistroInfo struct { + name string + manager distros.ITemplateManager + osDescription string + user string +} + +func (c *CloudBuilder) getExistingTemplateVersions() map[string]string { + templatePattern := regexp.MustCompile("^CSC (?P[A-Za-z][A-Za-z ]+[A-Za-z]) (?P\\d+(\\.\\d+)?)$") + mostRecentVersions := make(map[string]string) + templates := c.client.ListTemplates() + for _, template := range templates { + submatches := templatePattern.FindStringSubmatch(template.Name) + if submatches == nil { + log.Debug().Msg(fmt.Sprintf("Template '%s' did not match pattern, skipping", template.Name)) + continue + } + distro := submatches[1] + version := submatches[2] + log.Debug().Str("distro", distro).Str("version", version).Msg("Found template") + otherVersion, ok := mostRecentVersions[distro] + if !ok || version > otherVersion { + mostRecentVersions[distro] = version + } + } + return mostRecentVersions +} + +func (c *CloudBuilder) sendEmailNotification( + templateName string, vm *cloudstack.VirtualMachine, vmUser string, +) (err error) { + tmpl := template.Must(template.New("email-message").Parse(emailMessageTemplate)) + data := map[string]interface{}{ + "cfg": c.cfg, + "date": time.Now().Format(time.RFC1123Z), + "templateName": templateName, + "vm": vm, + "vmUser": vmUser, + } + var buf bytes.Buffer + if err = tmpl.Execute(&buf, data); err != nil { + return + } + // The lines of the body need to be CRLF terminated + // See https://pkg.go.dev/net/smtp#SendMail + // Make sure that email-message.txt uses Unix-style LF endings + msg := bytes.ReplaceAll(buf.Bytes(), []byte("\n"), []byte("\r\n")) + log.Debug(). + Str("to", c.cfg.EmailRecipient). + Msg("sending email notification") + return smtp.SendMail( + c.cfg.EmailServer, + nil, /* auth */ + c.cfg.EmailSender, + []string{c.cfg.EmailRecipient}, + msg, + ) +} + +func (c *CloudBuilder) createNewTemplate( + newVersion, codename string, distroInfo *DistroInfo, +) (template *cloudstack.Template, err error) { + distroManager := distroInfo.manager + templatePath, err := distroManager.DownloadTemplate(codename) + if err != nil { + return + } + if err = distroManager.ModifyTemplate(templatePath); err != nil { + return + } + templateName := fmt.Sprintf("CSC %s %s", distroInfo.name, newVersion) + template, err = c.UploadTemplate(templatePath, templateName, distroInfo.osDescription) + return +} + +func (c *CloudBuilder) versionStringCompare(version1, version2 string) int { + f1, err := strconv.ParseFloat(version1, 32) + if err != nil { + panic(err) + } + f2, err := strconv.ParseFloat(version2, 32) + if err != nil { + panic(err) + } + if f1 < f2 { + return -1 + } else if f1 > f2 { + return 1 + } + return 0 +} + +func (c *CloudBuilder) Start() (err error) { + distrosInfo := map[string]DistroInfo{ + "Ubuntu": DistroInfo{ + name: "Ubuntu", + manager: distros.NewUbuntuTemplateManager(c.cfg), + osDescription: "Other Ubuntu (64-bit)", + user: "ubuntu", + }, + "Fedora": DistroInfo{ + name: "Fedora", + manager: distros.NewFedoraTemplateManager(c.cfg), + osDescription: "Fedora Linux (64 bit)", + user: "fedora", + }, + "AlmaLinux": DistroInfo{ + name: "AlmaLinux", + manager: distros.NewAlmaLinuxTemplateManager(c.cfg), + osDescription: "Other CentOS (64-bit)", + user: "almalinux", + }, + } + // The elements of this slice must be keys of the map above + // TODO use environment variable + distrosToCheck := []string{"AlmaLinux"} + existingVersions := c.getExistingTemplateVersions() + for _, distro := range distrosToCheck { + distroInfo := distrosInfo[distro] + distroManager := distroInfo.manager + var newVersion, codename string + newVersion, codename, err = distroManager.GetLatestVersion() + if err != nil { + return + } + log.Debug().Str("newVersion", newVersion).Str("codename", codename).Msg(distro) + + curVersion, ok := existingVersions[distro] + if ok && c.versionStringCompare(newVersion, curVersion) <= 0 { + log.Debug(). + Str("distro", distro). + Str("version", curVersion). + Msg("Existing version is up to date, skipping") + continue + } + log.Info().Str("distro", distro).Msg("Existing template is out of date, creating a new one") + + var template *cloudstack.Template + template, err = c.createNewTemplate(newVersion, codename, &distroInfo) + if err != nil { + return + } + + vmName := strings.Join([]string{ + strings.ReplaceAll(strings.ToLower(distro), " ", "-"), + strings.ReplaceAll(newVersion, ".", "-"), + "test", + }, "-") + var vm *cloudstack.VirtualMachine + vm, err = c.CreateVM(vmName, template.Id) + if err != nil { + return + } + + if err = c.sendEmailNotification(template.Name, vm, distroInfo.user); err != nil { + return + } + } + return +} diff --git a/pkg/cloudbuilder/email-message.txt b/pkg/cloudbuilder/email-message.txt new file mode 100644 index 0000000..4c8645c --- /dev/null +++ b/pkg/cloudbuilder/email-message.txt @@ -0,0 +1,33 @@ +From: {{ .cfg.EmailSenderName }} <{{ .cfg.EmailSender }}> +To: {{ .cfg.EmailRecipient }} +Reply-To: {{ .cfg.EmailReplyTo }} +Subject: New VM Template: {{ .templateName }} +Date: {{ .date }} + +Hello syscom, + +This is an automated message from cloudbuild, the CSC VM template +builder. + +A new VM template, {{ .templateName }}, has been uploaded to CloudStack. +It is not public. A new VM, {{ .vm.Name }}, has been created from this +template. You can SSH into this VM from biloba or chamomile by running +the following: + + ssh -i /var/lib/cloudstack/management/.ssh/id_rsa {{ .vmUser }}@{{ (index .vm.Nic 0).IpAddress }} + +Please login to the VM and verify that everything is working correctly. +Once you have done this, please login to CloudStack with the admin account +and perform the following: + + 1. Delete the VM (enable the "Expunge" option too). + 2. Make the template public: + From the web page for the template, click the "Update Template Sharing" + circular button in the top right corner, and toggle the "Public" + and "Featured" sliders. + 3. Delete the old template (if it is not being used by any VMs): + From the web page for the old template, click the "Zones" tab, then + press the red circular "Delete" button beside "{{ .cfg.CloudstackZoneName }}". + +Sincerely, +cloudbuild diff --git a/pkg/cloudstack/cloudstack.go b/pkg/cloudstack/cloudstack.go new file mode 100644 index 0000000..0c09888 --- /dev/null +++ b/pkg/cloudstack/cloudstack.go @@ -0,0 +1,399 @@ +// Package cloudstack provides utilities for interacting with a CloudStack +// management server. +package cloudstack + +/* + * See https://cloudstack.apache.org/api/apidocs-4.16/ for API docs + */ + +import ( + "bytes" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + + "git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" +) + +// A CloudstackClient interacts with a CloudStack management server via the +// REST API. +type CloudstackClient struct { + cfg *config.Config + cachedZoneId string + cachedServiceOfferingId string +} + +// New returns a CloudStack client with the given config values. +func New(cfg *config.Config) *CloudstackClient { + return &CloudstackClient{cfg: cfg} +} + +// CloudStack uses java.net.URLEncoder when calculating signatures: +// https://github.com/apache/cloudstack/blob/main/server/src/main/java/com/cloud/api/ApiServer.java +// Unfortunately java.net.URLEncoder transforms "~" into "%7E". +// This is a bug: https://bugs.openjdk.org/browse/JDK-8204530 +// So, we need to match Java's behaviour so that the server does not +// reject our signature. +// (I noticed this problem when passing a `url` parameter to registerTemplate +// which contained a tilde.) +// TODO: file a GitHub issue in the cloudmonkey repo +func urlQueryEscape(value string) string { + return strings.ReplaceAll(url.QueryEscape(value), "~", "%7E") +} + +// Adapted from https://github.com/apache/cloudstack-cloudmonkey/blob/main/cmd/network.go +func encodeRequestParams(params map[string]string) string { + if params == nil { + return "" + } + + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sort.Strings(keys) + + var buf bytes.Buffer + for _, key := range keys { + value := params[key] + if buf.Len() > 0 { + buf.WriteByte('&') + } + buf.WriteString(key) + buf.WriteString("=") + buf.WriteString(urlQueryEscape(value)) + } + return buf.String() +} + +// See http://docs.cloudstack.apache.org/en/4.16.0.0/developersguide/dev.html#how-to-sign-an-api-call-with-python +func (client *CloudstackClient) createURL(params map[string]string) string { + cfg := client.cfg + params["apiKey"] = cfg.CloudstackApiKey + params["response"] = "json" + + // adapted from https://github.com/apache/cloudstack-cloudmonkey/blob/main/cmd/network.go + encodedParams := encodeRequestParams(params) + mac := hmac.New(sha1.New, []byte(cfg.CloudstackSecretKey)) + mac.Write([]byte(strings.Replace(strings.ToLower(encodedParams), "+", "%20", -1))) + signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) + encodedParams += fmt.Sprintf("&signature=%s", urlQueryEscape(signature)) + + return fmt.Sprintf("%s?%s", cfg.CloudstackApiBaseUrl, encodedParams) +} + +func getDeserializedResponse(url string, unmarshaledValue interface{}) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + panic(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + buf, err := io.ReadAll(resp.Body) + if err != nil { + panic(err) + } + if err = json.Unmarshal(buf, unmarshaledValue); err != nil { + panic(err) + } +} + +type ErrorInfo struct { + ErrorCode int `json:"errorcode"` + ErrorText string `json:"errortext"` +} + +func checkErrorInfo(data *ErrorInfo) { + if data.ErrorCode != 0 { + panic(fmt.Sprintf("Error %d: %s", data.ErrorCode, data.ErrorText)) + } +} + +type ListDomainsResponse struct { + ErrorInfo + Count int `json:"count"` + Domain []struct { + Name string `json:"name"` + Id string `json:"id"` + } `json:"domain"` +} + +// GetDomainId returns the ID of the domain with the given name. +func (client *CloudstackClient) GetDomainId(domainName string) string { + url := client.createURL(map[string]string{ + "command": "listDomains", + "details": "min", + "name": domainName, + }) + responseWrapper := struct { + Response ListDomainsResponse `json:"listdomainsresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + if data.Count != 1 { + panic("there should be one domain found") + } + return data.Domain[0].Id +} + +type Template struct { + Created string `json:"created"` + Domain string `json:"domain"` + DomainId string `json:"domainid"` + Id string `json:"id"` + Name string `json:"name"` + IsReady bool `json:"isready"` + IsFeatured bool `json:"isfeatured"` + IsPublic bool `json:"ispublic"` + Status string `json:"status"` + DownloadDetails []struct { + DownloadPercent string `json:"downloadPercent"` + DownloadState string `json:"downloadState"` + } `json:"downloaddetails"` +} + +type ListTemplatesResponse struct { + ErrorInfo + Count int `json:"count"` + Template []Template `json:"template"` +} + +func (client *CloudstackClient) listTemplates(templateID string) []Template { + params := map[string]string{ + "command": "listTemplates", + "details": "min", + "templatefilter": "self", + } + if templateID != "" { + params["id"] = templateID + } + url := client.createURL(params) + responseWrapper := struct { + Response ListTemplatesResponse `json:"listtemplatesresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + return data.Template +} + +func (client *CloudstackClient) ListTemplates() []Template { + return client.listTemplates("") +} + +func (client *CloudstackClient) GetTemplate(templateID string) (*Template, error) { + templates := client.listTemplates(templateID) + if len(templates) == 0 { + return nil, errors.New("Template not found") + } + return &templates[0], nil +} + +type ListZonesResponse struct { + ErrorInfo + Count int `json:"count"` + Zone []struct { + Id string `json:"id"` + Name string `json:"name"` + } `json:"zone"` +} + +func (client *CloudstackClient) getZoneId() (string, error) { + if client.cachedZoneId != "" { + return client.cachedZoneId, nil + } + zoneName := client.cfg.CloudstackZoneName + url := client.createURL(map[string]string{ + "command": "listZones", + "name": zoneName, + }) + responseWrapper := struct { + Response ListZonesResponse `json:"listzonesresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + if data.Count != 1 { + return "", errors.New(fmt.Sprintf("Expected 1 zone for '%s'; got %d", zoneName, data.Count)) + } + zoneId := data.Zone[0].Id + client.cachedZoneId = zoneId + return zoneId, nil +} + +type ListServiceOfferingsResponse struct { + ErrorInfo + Count int `json:"count"` + ServiceOffering []struct { + Id string `json:"id"` + Name string `json:"name"` + } `json:"serviceoffering"` +} + +func (client *CloudstackClient) getServiceOfferingId() (string, error) { + if client.cachedServiceOfferingId != "" { + return client.cachedServiceOfferingId, nil + } + serviceOfferingName := client.cfg.CloudstackServiceOfferingName + url := client.createURL(map[string]string{ + "command": "listServiceOfferings", + "name": serviceOfferingName, + }) + responseWrapper := struct { + Response ListServiceOfferingsResponse `json:"listserviceofferingsresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + if data.Count != 1 { + return "", errors.New(fmt.Sprintf("Expected 1 service offering for '%s'; got %d", serviceOfferingName, data.Count)) + } + serviceOfferingId := data.ServiceOffering[0].Id + client.cachedServiceOfferingId = serviceOfferingId + return serviceOfferingId, nil +} + +// Note: if an OS type you need isn't available in CloudStack, you can +// create a new one with addGuestOs + +type OsType struct { + Id string `json:"id"` + Description string `json:"description"` +} + +type ListOsTypesResponse struct { + ErrorInfo + Count int `json:"count"` + OsType []OsType `json:"ostype"` +} + +func (client *CloudstackClient) GetOsTypeId(osDescription string) (string, error) { + url := client.createURL(map[string]string{ + "command": "listOsTypes", + "description": osDescription, + }) + responseWrapper := struct { + Response ListOsTypesResponse `json:"listostypesresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + if data.Count != 1 { + return "", errors.New(fmt.Sprintf("Expected 1 OS type for '%s'; got %d", osDescription, data.Count)) + } + return data.OsType[0].Id, nil +} + +type RegisterTemplateResponse struct { + ErrorInfo + Count int `json:"count"` + Template []Template `json:"template"` +} + +func (client *CloudstackClient) RegisterTemplate(name string, downloadUrl string, osTypeId string) (*Template, error) { + zoneId, err := client.getZoneId() + if err != nil { + return nil, err + } + url := client.createURL(map[string]string{ + "command": "registerTemplate", + "displaytext": name, + "format": "QCOW2", + "hypervisor": "KVM", + "name": name, + "url": downloadUrl, + "isextractable": "true", + "isfeatured": "false", + "ispublic": "false", + "ostypeid": osTypeId, + "requireshvm": "true", + "zoneid": zoneId, + }) + responseWrapper := struct { + Response RegisterTemplateResponse `json:"registertemplateresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + return &data.Template[0], nil +} + +type VirtualMachine struct { + Id string `json:"id"` + DisplayName string `json:"displayname"` + Name string `json:"name"` + State string `json:"state"` + Nic []struct { + Id string `json:"id"` + IpAddress string `json:"ipaddress"` + IsDefault bool `json:"isdefault"` + } `json:"nic"` +} + +type DeployVirtualMachineResponse struct { + ErrorInfo + Id string `json:"id"` + JobId string `json:"jobid"` +} + +func (client *CloudstackClient) DeployVirtualMachine( + name string, templateID string, +) (vmID string, err error) { + serviceOfferingId, err := client.getServiceOfferingId() + if err != nil { + return + } + zoneId, err := client.getZoneId() + if err != nil { + return + } + url := client.createURL(map[string]string{ + "command": "deployVirtualMachine", + "serviceofferingid": serviceOfferingId, + "templateid": templateID, + "zoneid": zoneId, + "name": name, + "displayname": name, + "keypair": client.cfg.CloudstackKeypairName, + }) + responseWrapper := struct { + Response DeployVirtualMachineResponse `json:"deployvirtualmachineresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + return data.Id, nil +} + +type ListVirtualMachinesResponse struct { + ErrorInfo + Count int `json:"count"` + VirtualMachine []VirtualMachine `json:"virtualmachine"` +} + +func (client *CloudstackClient) GetVirtualMachine(vmID string) (virtualMachine *VirtualMachine, err error) { + url := client.createURL(map[string]string{ + "command": "listVirtualMachines", + "id": vmID, + }) + responseWrapper := struct { + Response ListVirtualMachinesResponse `json:"listvirtualmachinesresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + virtualMachine = &data.VirtualMachine[0] + return +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..4ae891e --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,62 @@ +// Package config provides utilities for reading and storing configuration +// values. +package config + +import ( + "os" +) + +// A Config holds all of the configuration values needed for the program. +type Config struct { + CloudstackApiKey string + CloudstackSecretKey string + CloudstackApiBaseUrl string + CloudstackZoneName string + CloudstackServiceOfferingName string + CloudstackKeypairName string + UploadDirectory string + UploadBaseUrl string + MirrorHost string + EmailServer string + EmailSender string + EmailSenderName string + EmailRecipient string + EmailReplyTo string +} + +// New returns a Config filled with values read from environment variables. +// It panics if a required environment variable is empty or not set. +func New() *Config { + cfg := &Config{ + CloudstackApiKey: os.Getenv("CLOUDSTACK_API_KEY"), + CloudstackSecretKey: os.Getenv("CLOUDSTACK_SECRET_KEY"), + UploadDirectory: os.Getenv("UPLOAD_DIRECTORY"), + UploadBaseUrl: os.Getenv("UPLOAD_BASE_URL"), + } + if cfg.CloudstackApiKey == "" { + panic("CLOUDSTACK_API_KEY is empty or not set") + } + if cfg.CloudstackSecretKey == "" { + panic("CLOUDSTACK_SECRET_KEY is empty or not set") + } + if cfg.UploadDirectory == "" { + panic("UPLOAD_DIRECTORY is empty or not set") + } + if cfg.UploadBaseUrl == "" { + panic("UPLOAD_BASE_URL is empty or not set") + } + + // These should never change + cfg.CloudstackApiBaseUrl = "https://cloud.csclub.uwaterloo.ca/client/api" + cfg.CloudstackZoneName = "Zone1" + cfg.CloudstackServiceOfferingName = "Small Instance" + cfg.CloudstackKeypairName = "management-keypair" + cfg.MirrorHost = "mirror.csclub.uwaterloo.ca" + cfg.EmailServer = "mail.csclub.uwaterloo.ca:25" + cfg.EmailSender = "cloudbuild@csclub.uwaterloo.ca" + cfg.EmailSenderName = "cloudbuild" + // TODO: change recipient to syscom + cfg.EmailRecipient = "merenber@csclub.uwaterloo.ca" + cfg.EmailReplyTo = "no-reply@csclub.uwaterloo.ca" + return cfg +} diff --git a/pkg/distros/almalinux.go b/pkg/distros/almalinux.go new file mode 100644 index 0000000..0691924 --- /dev/null +++ b/pkg/distros/almalinux.go @@ -0,0 +1,139 @@ +package distros + +import ( + "errors" + "fmt" + "math" + "net/http" + "regexp" + "strconv" + + "github.com/rs/zerolog/log" + "golang.org/x/net/html" + "libguestfs.org/guestfs" + + "git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" +) + +// TODO: reduce code duplication with FedoraTemplateManager + +type AlmaLinuxTemplateManager struct { + TemplateManager +} + +// NewAlmaLinuxTemplateManager returns a AlmaLinuxTemplateManager with the given +// config values. +func NewAlmaLinuxTemplateManager(cfg *config.Config) *AlmaLinuxTemplateManager { + logger := log.With().Str("distro", "almalinux").Logger() + almaLinuxTemplateManager := AlmaLinuxTemplateManager{ + TemplateManager{ + cfg: cfg, + logger: &logger, + impl: nil, + }, + } + almaLinuxTemplateManager.TemplateManager.impl = &almaLinuxTemplateManager + return &almaLinuxTemplateManager +} + +var floatNumberSlashPattern *regexp.Regexp = regexp.MustCompile("^\\d+(\\.\\d)?/$") + +func getMaxFloatVersionFromHtml(node *html.Node, maxVersion *float64) { + if isHyperlink(node) { + text := node.FirstChild.Data + if numberSlashPattern.FindString(text) != "" { + version, _ := strconv.ParseFloat(text[:len(text)-1], 64) + if version > *maxVersion { + *maxVersion = version + } + } + return + } + for c := node.FirstChild; c != nil; c = c.NextSibling { + getMaxFloatVersionFromHtml(c, maxVersion) + } +} + +func (mgr *AlmaLinuxTemplateManager) GetLatestVersion() (version string, codename string, err error) { + resp, err := http.Get("https://mirror.csclub.uwaterloo.ca/almalinux/") + if err != nil { + return + } + defer resp.Body.Close() + node, err := html.Parse(resp.Body) + if err != nil { + return + } + var floatVersion float64 + getMaxFloatVersionFromHtml(node, &floatVersion) + if floatVersion == 0 { + err = errors.New("Could not determine latest AlmaLinux version from HTML") + } else { + version = strconv.FormatFloat(floatVersion, 'f', -1, 64) + // AlmaLinux doesn't have codenames, only numbered versions + codename = version + } + return +} + +func (mgr *AlmaLinuxTemplateManager) DownloadTemplate(codename string) (path string, err error) { + // TODO: REMOVE THIS + //if true { + // return "AlmaLinux-9-GenericCloud-latest.x86_64.qcow2", nil + //} + version := codename + floatVersion, err := strconv.ParseFloat(version, 64) + if err != nil { + return + } + majorVersion := int(math.Floor(floatVersion)) + filename := fmt.Sprintf("AlmaLinux-%d-GenericCloud-latest.x86_64.qcow2", majorVersion) + url := fmt.Sprintf("https://mirror.csclub.uwaterloo.ca/almalinux/%s/cloud/x86_64/images/%s", version, filename) + return mgr.DownloadTemplateGeneric(filename, url) +} + +func (mgr *AlmaLinuxTemplateManager) GetIpv6ScriptsTemplateData() map[string]string { + return map[string]string{ + "networkService": "NetworkManager.service", + "networkTarget": "network-online.target", + } +} + +func (mgr *AlmaLinuxTemplateManager) addCloudInitSnippet(handle *guestfs.Guestfs) error { + path := "/etc/cloud/cloud.cfg.d/99_csclub.cfg" + mgr.logger.Debug().Msg("Writing to " + path) + return handle.Write(path, getResource("fedora-cloud-init")) +} + +func (mgr *AlmaLinuxTemplateManager) HasSELinuxEnabled() bool { + return true +} + +var almaLinuxYumRepoBaseUrlPattern *regexp.Regexp = regexp.MustCompile( + "^(?Phttps?://)[A-Za-z0-9./-]+(?P/almalinux/\\$releasever/[A-Za-z0-9./$-]+)$", +) + +func (mgr *AlmaLinuxTemplateManager) transformAlmaLinuxYumRepoBaseUrl(url string) string { + submatches := almaLinuxYumRepoBaseUrlPattern.FindStringSubmatch(url) + if submatches != nil { + scheme, path := submatches[1], submatches[2] + url = scheme + mgr.cfg.MirrorHost + path + } + return url +} + +func (mgr *AlmaLinuxTemplateManager) PerformDistroSpecificModifications(handle *guestfs.Guestfs) (err error) { + if err = mgr.setChronyOptions(handle, "/etc/chrony.conf"); err != nil { + return + } + if err = mgr.setNetworkManagerOptions(handle); err != nil { + return + } + if err = mgr.addCloudInitSnippet(handle); err != nil { + return + } + if err = mgr.replaceYumRepoMirrorUrls(handle, mgr.transformAlmaLinuxYumRepoBaseUrl); err != nil { + return + } + return +} diff --git a/pkg/distros/fedora.go b/pkg/distros/fedora.go new file mode 100644 index 0000000..11b69fb --- /dev/null +++ b/pkg/distros/fedora.go @@ -0,0 +1,161 @@ +package distros + +import ( + "errors" + "fmt" + "net/http" + "regexp" + "strconv" + + "github.com/rs/zerolog/log" + "golang.org/x/net/html" + "libguestfs.org/guestfs" + + "git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" +) + +type FedoraTemplateManager struct { + TemplateManager +} + +// NewFedoraTemplateManager returns a FedoraTemplateManager with the given +// config values. +func NewFedoraTemplateManager(cfg *config.Config) *FedoraTemplateManager { + logger := log.With().Str("distro", "fedora").Logger() + fedoraTemplateManager := FedoraTemplateManager{ + TemplateManager{ + cfg: cfg, + logger: &logger, + impl: nil, + }, + } + fedoraTemplateManager.TemplateManager.impl = &fedoraTemplateManager + return &fedoraTemplateManager +} + +var numberSlashPattern *regexp.Regexp = regexp.MustCompile("^\\d+/$") + +func isHyperlink(node *html.Node) bool { + return node.Type == html.ElementNode && node.Data == "a" && + node.FirstChild != nil && node.FirstChild.Type == html.TextNode +} + +func getMatchingLinkFromHtml(node *html.Node, pattern *regexp.Regexp) string { + if isHyperlink(node) { + text := node.FirstChild.Data + return pattern.FindString(text) + } + for c := node.FirstChild; c != nil; c = c.NextSibling { + result := getMatchingLinkFromHtml(c, pattern) + if result != "" { + return result + } + } + return "" +} + +func getMaxVersionFromHtml(node *html.Node, maxVersion *int64) { + if isHyperlink(node) { + text := node.FirstChild.Data + if numberSlashPattern.FindString(text) != "" { + version, _ := strconv.ParseInt(text[:len(text)-1], 10, 64) + if version > *maxVersion { + *maxVersion = version + } + } + return + } + for c := node.FirstChild; c != nil; c = c.NextSibling { + getMaxVersionFromHtml(c, maxVersion) + } +} + +func (mgr *FedoraTemplateManager) GetLatestVersion() (version string, codename string, err error) { + resp, err := http.Get("https://mirror.csclub.uwaterloo.ca/fedora/linux/releases/") + if err != nil { + return + } + defer resp.Body.Close() + node, err := html.Parse(resp.Body) + if err != nil { + return + } + var intVersion int64 + getMaxVersionFromHtml(node, &intVersion) + if intVersion == 0 { + err = errors.New("Could not determine latest Fedora version from HTML") + } else { + version = strconv.FormatInt(intVersion, 10) + // Fedora doesn't have codenames, only numbered versions + codename = version + } + return +} + +func (mgr *FedoraTemplateManager) DownloadTemplate(codename string) (path string, err error) { + version := codename + pattern := regexp.MustCompile(fmt.Sprintf("^Fedora-Cloud-Base-%s-\\d+(\\.\\d+)?\\.x86_64\\.qcow2$", version)) + imagesUrl := fmt.Sprintf("https://mirror.csclub.uwaterloo.ca/fedora/linux/releases/%s/Cloud/x86_64/images/", version) + resp, err := http.Get(imagesUrl) + if err != nil { + return + } + defer resp.Body.Close() + node, err := html.Parse(resp.Body) + if err != nil { + return + } + filename := getMatchingLinkFromHtml(node, pattern) + if filename == "" { + err = errors.New("Could not find hyperlink for Fedora Cloud image") + return + } + url := imagesUrl + filename + return mgr.DownloadTemplateGeneric(filename, url) +} + +func (mgr *FedoraTemplateManager) GetIpv6ScriptsTemplateData() map[string]string { + return map[string]string{ + "networkService": "NetworkManager.service", + "networkTarget": "network-online.target", + } +} + +func (mgr *FedoraTemplateManager) addCloudInitSnippet(handle *guestfs.Guestfs) error { + path := "/etc/cloud/cloud.cfg.d/99_csclub.cfg" + mgr.logger.Debug().Msg("Writing to " + path) + return handle.Write(path, getResource("fedora-cloud-init")) +} + +func (mgr *FedoraTemplateManager) HasSELinuxEnabled() bool { + return true +} + +var fedoraYumRepoBaseUrlPattern *regexp.Regexp = regexp.MustCompile( + "^(?Phttps?://)[A-Za-z0-9./-]+(?P/fedora/linux/[A-Za-z0-9./$-]+)$", +) + +func (mgr *FedoraTemplateManager) transformFedoraYumRepoBaseUrl(url string) string { + submatches := fedoraYumRepoBaseUrlPattern.FindStringSubmatch(url) + if submatches != nil { + scheme, path := submatches[1], submatches[2] + url = scheme + mgr.cfg.MirrorHost + path + } + return url +} + +func (mgr *FedoraTemplateManager) PerformDistroSpecificModifications(handle *guestfs.Guestfs) (err error) { + if err = mgr.setChronyOptions(handle, "/etc/chrony.conf"); err != nil { + return + } + if err = mgr.setNetworkManagerOptions(handle); err != nil { + return + } + if err = mgr.addCloudInitSnippet(handle); err != nil { + return + } + if err = mgr.replaceYumRepoMirrorUrls(handle, mgr.transformFedoraYumRepoBaseUrl); err != nil { + return + } + return +} diff --git a/pkg/distros/resources/98_csclub_disable_nm_ipv6.sh b/pkg/distros/resources/98_csclub_disable_nm_ipv6.sh new file mode 100644 index 0000000..9b01c95 --- /dev/null +++ b/pkg/distros/resources/98_csclub_disable_nm_ipv6.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -ex + +nmcli -f name -terse c | grep "^cloud-init e" | \ + while read conn_name; do + iface=$(echo "$conn_name" | cut -d ' ' -f 2) + nmcli c modify "$conn_name" ipv6.method link-local + nmcli d reapply $iface + done + +ip -6 -brief addr show dynamic | \ + while read iface _ addrs; do + for addr in $addrs; do + ip addr del dev $iface $addr + done + done + +ip -6 route show proto ra | \ + while read line; do + ip route del $line + done diff --git a/pkg/distros/resources/99_csclub_ipv6_addr.sh b/pkg/distros/resources/99_csclub_ipv6_addr.sh new file mode 100644 index 0000000..ac6e143 --- /dev/null +++ b/pkg/distros/resources/99_csclub_ipv6_addr.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -ex +INTERFACE= +IPV4_ADDRESS= +while read interface _ address; do + if ! echo $address | grep -q '^172\.19\.134\.'; then + continue + fi + INTERFACE=$interface + IPV4_ADDRESS=$address + break +done < <(ip -4 -brief addr show) +if [ -z "$INTERFACE" ]; then + echo "Could not find primary interface" >&2 + exit 1 +fi +NUM=$(echo $IPV4_ADDRESS | grep -oP '^172\.19\.134\.\K(\d+)') +IPV6_ADDRESS="2620:101:f000:4903::$NUM/64" +ip -6 addr add $IPV6_ADDRESS dev $INTERFACE +ip -6 route add default via 2620:101:f000:4903::1 dev $INTERFACE diff --git a/pkg/distros/resources/chrony-snippet b/pkg/distros/resources/chrony-snippet new file mode 100644 index 0000000..90997e4 --- /dev/null +++ b/pkg/distros/resources/chrony-snippet @@ -0,0 +1,8 @@ +#server ntp.csclub.uwaterloo.ca +server 129.97.167.12 +#server ntp.student.cs.uwaterloo.ca +server 129.97.167.4 +#server ntp.cs.uwaterloo.ca +server 129.97.15.14 +#server ntp.cscf.uwaterloo.ca +server 129.97.15.15 diff --git a/pkg/distros/resources/fedora-cloud-init b/pkg/distros/resources/fedora-cloud-init new file mode 100644 index 0000000..3f478b5 --- /dev/null +++ b/pkg/distros/resources/fedora-cloud-init @@ -0,0 +1 @@ +manage_etc_hosts: true diff --git a/pkg/distros/resources/ipv6-sysctl.sh b/pkg/distros/resources/ipv6-sysctl.sh new file mode 100644 index 0000000..ac1d4fa --- /dev/null +++ b/pkg/distros/resources/ipv6-sysctl.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -ex +while read interface _; do + if [ $interface = lo ]; then + continue + fi + sysctl net.ipv6.conf.$interface.accept_ra=0 + sysctl net.ipv6.conf.$interface.accept_dad=0 +done < <(ip -brief link show) diff --git a/pkg/distros/resources/motd b/pkg/distros/resources/motd new file mode 100644 index 0000000..9763a2c --- /dev/null +++ b/pkg/distros/resources/motd @@ -0,0 +1,12 @@ + +UNIVERSITY OF WATERLOO COMPUTER SCIENCE CLUB +Welcome to the CSC cloud! :) +This service was made possible thanks to donations from the Mathematics +Endowment Fund (MEF) and the Computer Science Computing Facility (CSCF). + +Documentation: + https://docs.cloud.csclub.uwaterloo.ca +Machine usage agreement: + https://csclub.uwaterloo.ca/resources/machine-usage-agreement +Questions/concerns can be emailed to: + syscom@csclub.uwaterloo.ca diff --git a/pkg/distros/resources/network-manager-snippet b/pkg/distros/resources/network-manager-snippet new file mode 100644 index 0000000..c338485 --- /dev/null +++ b/pkg/distros/resources/network-manager-snippet @@ -0,0 +1,3 @@ +[main] +dhcp = dhclient +dns = none diff --git a/pkg/distros/resources/resolv.conf b/pkg/distros/resources/resolv.conf new file mode 100644 index 0000000..fa815a8 --- /dev/null +++ b/pkg/distros/resources/resolv.conf @@ -0,0 +1,12 @@ +search csclub.uwaterloo.ca uwaterloo.ca +options rotate timeout:1 attempts:1 ndots:2 + +# CSC Nameservers +nameserver 2620:101:f000:4901:c5c::4 +nameserver 2620:101:f000:7300:c5c::20 +nameserver 129.97.134.4 +nameserver 129.97.18.20 + +# IST Anycast (fallback) +#nameserver 129.97.2.1 +#nameserver 129.97.2.2) diff --git a/pkg/distros/resources/sysctl.conf b/pkg/distros/resources/sysctl.conf new file mode 100644 index 0000000..55abb91 --- /dev/null +++ b/pkg/distros/resources/sysctl.conf @@ -0,0 +1,2 @@ +net.ipv6.conf.all.accept_ra = 0 +net.ipv6.conf.default.accept_ra = 0 diff --git a/pkg/distros/resources/timesyncd.conf b/pkg/distros/resources/timesyncd.conf new file mode 100644 index 0000000..c443e2b --- /dev/null +++ b/pkg/distros/resources/timesyncd.conf @@ -0,0 +1,5 @@ +[Time] +# ntp.csclub.uwaterloo.ca ntp.student.cs.uwaterloo.ca +NTP=129.97.167.12 129.97.167.4 +# ntp.cs.uwaterloo.ca ntp.cscf.uwaterloo.ca +FallbackNTP=129.97.15.14 129.97.15.15 diff --git a/pkg/distros/resources/ubuntu-cloud-init b/pkg/distros/resources/ubuntu-cloud-init new file mode 100644 index 0000000..5a13e97 --- /dev/null +++ b/pkg/distros/resources/ubuntu-cloud-init @@ -0,0 +1,11 @@ +network: + version: 2 + ethernets: + id0: + match: + name: e* + dhcp4: true + accept-ra: false + +apt_preserve_sources_list: true +manage_etc_hosts: true diff --git a/pkg/distros/template_manager.go b/pkg/distros/template_manager.go new file mode 100644 index 0000000..c4412ff --- /dev/null +++ b/pkg/distros/template_manager.go @@ -0,0 +1,526 @@ +package distros + +import ( + "embed" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "text/template" + + "github.com/rs/zerolog" + "libguestfs.org/guestfs" + + "git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" +) + +//go:embed resources +var res embed.FS + +func getResource(filename string) []byte { + data, err := res.ReadFile("resources/" + filename) + if err != nil { + panic(err) + } + return data +} + +func getTemplateResource(filename string) *template.Template { + tmpl, err := template.ParseFS(res, "resources/"+filename) + if err != nil { + panic(err) + } + return tmpl +} + +// A TemplateManager downloads and modifies VM templates for a distro. +type TemplateManager struct { + cfg *config.Config + logger *zerolog.Logger + impl IDistroSpecificTemplateManager +} + +type ITemplateManager interface { + // GetLatestVersion returns the version number and codename of the + // latest version of particular OS (e.g. version="22.04", codename="jammy") + GetLatestVersion() (version string, codename string, err error) + // DownloadTemplate downloads a VM template for a given OS codename + // and returns the path to where it was downloaded + DownloadTemplate(codename string) (path string, err error) + // ModifyTemplate makes custom modifications to the downloaded VM template + ModifyTemplate(filename string) error +} + +// An IDistroSpecificTemplateManager performs the distro-specific tasks +// when modifying a VM template. It is used by the generic TemplateManager. +type IDistroSpecificTemplateManager interface { + // GetIpv6ScriptsTemplateData returns template data used by the + // IPv6 scripts. The required keys are "networkService" and + // "networkTarget"; the values should be a systemd service and target, + // respectively. + GetIpv6ScriptsTemplateData() map[string]string + // PerformDistroSpecificModifications is called after + // performDistroAgnosticModifications to modify a template in a + // distro-specific way. + PerformDistroSpecificModifications(handle *guestfs.Guestfs) error + // HasSELinuxEnabled returns whether SELinux is enabled for this distro + HasSELinuxEnabled() bool +} + +func (mgr *TemplateManager) DownloadTemplateGeneric(filename, url string) (path string, err error) { + mgr.logger.Debug().Str("url", url).Msg("Downloading template") + resp, err := http.Get(url) + if err != nil { + return + } + defer resp.Body.Close() + // TODO: download to somewhere in /tmp + path = filename + out, err := os.Create(path) + if err != nil { + return + } + defer out.Close() + _, err = io.Copy(out, resp.Body) + return +} + +func (mgr *TemplateManager) performDistroAgnosticModifications(handle *guestfs.Guestfs) (err error) { + templateData := mgr.impl.GetIpv6ScriptsTemplateData() + if err = mgr.setupIpv6Scripts(handle, templateData); err != nil { + return + } + if err = mgr.setResolvConf(handle); err != nil { + return + } + if err = mgr.setMotd(handle); err != nil { + return + } + if err = mgr.updateSshdConfig(handle); err != nil { + return + } + return +} + +func (mgr *TemplateManager) ModifyTemplate(filename string) (err error) { + handle, err := mgr.getGuestfsMountedHandle(filename) + if err != nil { + return + } + defer mgr.unmountAndCloseGuestfsHandle(handle) + if err = mgr.createAugeasHandle(handle); err != nil { + return + } + defer mgr.closeAugeasHandle(handle) + if err = mgr.performDistroAgnosticModifications(handle); err != nil { + return + } + if err = mgr.impl.PerformDistroSpecificModifications(handle); err != nil { + return + } + if mgr.impl.HasSELinuxEnabled() { + if err = mgr.selinuxRelabelDirectories(handle); err != nil { + return + } + } + return mgr.saveAugeasValues(handle) +} + +func (mgr *TemplateManager) getGuestfsMountedHandle(filename string) (handle *guestfs.Guestfs, err error) { + log := mgr.logger + handle, err = guestfs.Create() + if err != nil { + return + } + log.Debug().Msg("Adding drive " + filename) + if err = handle.Add_drive(filename, nil); err != nil { + return + } + log.Debug().Msg("Launching VM") + if err = handle.Launch(); err != nil { + return + } + log.Debug().Msg("Inspecting OS") + partitions, err := handle.Inspect_os() + if err != nil { + return + } + if len(partitions) != 1 { + return nil, errors.New(fmt.Sprintf("Expected 1 root partition, found %d", len(partitions))) + } + rootPartition := partitions[0] + log.Debug().Msg(fmt.Sprintf("Mounting root filesystem %s on /", rootPartition)) + if err = handle.Mount(rootPartition, "/"); err != nil { + return + } + return +} + +func (mgr *TemplateManager) unmountGuestfsDrive(handle *guestfs.Guestfs) { + if err := handle.Umount("/", nil); err != nil { + mgr.logger.Error().Err(err).Msg("") + } +} + +func (mgr *TemplateManager) unmountAndCloseGuestfsHandle(handle *guestfs.Guestfs) { + mgr.unmountGuestfsDrive(handle) + handle.Close() +} + +func getSelinuxType(handle *guestfs.Guestfs) (selinuxType string, err error) { + lines, err := handle.Grep("^SELINUXTYPE=", "/etc/selinux/config", nil) + if err != nil { + return + } + if len(lines) != 1 { + err = errors.New(fmt.Sprintf("Expected 1 line containing SELINUXTYPE, found %d", len(lines))) + return + } + selinuxType = strings.Split(lines[0], "=")[1] + return +} + +func getSelinuxDefaultSpecfile(handle *guestfs.Guestfs) (specfile string, err error) { + selinuxType, err := getSelinuxType(handle) + if err != nil { + return + } + specfile = fmt.Sprintf("/etc/selinux/%s/contexts/files/file_contexts", selinuxType) + return +} + +func (mgr *TemplateManager) selinuxRelabel(handle *guestfs.Guestfs, specfile, dir string) (err error) { + mgr.logger.Debug(). + Str("specfile", specfile). + Str("path", dir). + Msg("Relabeling SELinux security context") + return handle.Selinux_relabel(specfile, dir, nil) +} + +func (mgr *TemplateManager) selinuxRelabelDirectories(handle *guestfs.Guestfs) (err error) { + specfile, err := getSelinuxDefaultSpecfile(handle) + if err != nil { + return + } + for _, dir := range []string{"/etc", "/var/lib/cloud"} { + if err = mgr.selinuxRelabel(handle, specfile, dir); err != nil { + return + } + } + return +} + +// setChronyOptions sets custom NTP server URLs in a chrony config file. +// It assumes that a line beginning with "pool" will already be present. +func (mgr *TemplateManager) setChronyOptions(handle *guestfs.Guestfs, path string) (err error) { + oldLines, err := handle.Read_lines(path) + if err != nil { + return + } + snippet := string(getResource("chrony-snippet")) + snippetLines := strings.Split(snippet, "\n") + wroteSnippet := false + newLines := make([]string, 0, len(oldLines)+len(snippetLines)) + for _, line := range oldLines { + if strings.HasPrefix(line, "pool ") { + newLines = append(newLines, "#"+line) + if !wroteSnippet { + newLines = append(newLines, snippetLines...) + wroteSnippet = true + } + } else { + newLines = append(newLines, line) + } + } + newContent := strings.Join(newLines, "\n") + mgr.logger.Debug().Msg("Writing new content to " + path) + return handle.Write(path, []byte(newContent)) +} + +func (mgr *TemplateManager) setNetworkManagerOptions(handle *guestfs.Guestfs) (err error) { + if err = handle.Mkdir_p("/etc/NetworkManager/conf.d"); err != nil { + return + } + path := "/etc/NetworkManager/conf.d/99_csclub.conf" + mgr.logger.Debug().Msg("Writing to " + path) + return handle.Write(path, getResource("network-manager-snippet")) +} + +func (mgr *TemplateManager) setupIpv6Scripts(handle *guestfs.Guestfs, templateData map[string]string) (err error) { + log := mgr.logger + scripts := []string{"99_csclub_ipv6_addr.sh"} + if templateData["networkService"] == "NetworkManager.service" { + scripts = append(scripts, "98_csclub_disable_nm_ipv6.sh") + } + scriptDir := "/var/lib/cloud/scripts/per-boot" + if err = handle.Mkdir_p(scriptDir); err != nil { + return + } + for _, filename := range scripts { + path := scriptDir + "/" + filename + log.Debug().Msg("Writing to " + path) + if err = handle.Write(path, getResource(filename)); err != nil { + return + } + if err = handle.Chmod(0755, path); err != nil { + return + } + } + sysctlPath := "/etc/sysctl.d/csclub.conf" + log.Debug().Msg("Writing to " + sysctlPath) + return handle.Write(sysctlPath, getResource("sysctl.conf")) +} + +func (mgr *TemplateManager) setResolvConf(handle *guestfs.Guestfs) (err error) { + mgr.logger.Debug().Msg("Writing to /etc/resolv.conf") + if err = handle.Rm_f("/etc/resolv.conf"); err != nil { + return + } + return handle.Write("/etc/resolv.conf", getResource("resolv.conf")) +} + +func (mgr *TemplateManager) createAugeasHandle(handle *guestfs.Guestfs) (err error) { + mgr.logger.Debug().Msg("Creating a new Augeas handle") + return handle.Aug_init("/", 0) +} + +func (mgr *TemplateManager) closeAugeasHandle(handle *guestfs.Guestfs) { + if err := handle.Aug_close(); err != nil { + mgr.logger.Error().Err(err).Msg("") + } +} + +func (mgr *TemplateManager) saveAugeasValues(handle *guestfs.Guestfs) error { + mgr.logger.Debug().Msg("Saving Augeas values") + return handle.Aug_save() +} + +// requires an Augeas handle to be open +func (mgr *TemplateManager) setDhclientOptions(handle *guestfs.Guestfs) (err error) { + mgr.logger.Debug().Msg("Retrieving dhclient request options") + dhclientRequestOptionNodes, err := handle.Aug_ls("/files/etc/dhcp/dhclient.conf/request") + if err != nil { + return + } + optionsToRemove := map[string]bool{ + "domain-name": true, + "domain-name-servers": true, + "domain-search": true, + "dhcp6.name-servers": true, + "dhcp6.domain-search": true, + "dhcp6.fqdn": true, + "dhcp6.sntp-servers": true, + "ntp-servers": true, + "netbios-name-servers": true, + } + for _, optionPath := range dhclientRequestOptionNodes { + // optionPath will look something like /files/etc/dhcp/dhclient.conf/request/5 + var optionValue string + optionValue, err = handle.Aug_get(optionPath) + if err != nil { + return + } + if _, ok := optionsToRemove[optionValue]; !ok { + continue + } + mgr.logger.Debug().Msg("Removing dhclient request option " + optionValue) + if _, err = handle.Aug_rm(optionPath); err != nil { + return + } + } + return +} + +func getNamedRegexGroup(re *regexp.Regexp, submatches []string, groupName string) string { + var value string + for i, subexpName := range re.SubexpNames() { + if subexpName == groupName { + value = submatches[i] + break + } + } + if value == "" { + panic("Could not find regex group " + groupName) + } + return value +} + +func (mgr *TemplateManager) addAugeasComment( + handle *guestfs.Guestfs, numExistingComments int, parentNode string, comment string, +) error { + return handle.Aug_set( + parentNode+fmt.Sprintf("/#comment[%d]", numExistingComments), + comment, + ) +} + +// replaceYumRepoMirrorUrls comments out the metalink URLs and uncomments and +// replaces the baseurl URLs for each repo in /etc/yum.repos.d. +// It assumes that the baseurl will be a commented line. +// +// transformBaseUrl accepts a repo base URL and replaces the host (and +// optionally parts of the path) as appropriate. It should return the input +// if the URL should not be modified. +func (mgr *TemplateManager) replaceYumRepoMirrorUrls( + handle *guestfs.Guestfs, transformBaseurl func(string) string, +) (err error) { + log := mgr.logger + repoPaths, err := handle.Aug_ls("/files/etc/yum.repos.d") + if err != nil { + return fmt.Errorf("Could not enumerate yum repos: %w", err) + } + // A repoPath looks like e.g. /files/etc/yum.repos.d/almalinux-appstream.repo + for _, repoPath := range repoPaths { + subrepoPaths, err := handle.Aug_ls(repoPath) + if err != nil { + return fmt.Errorf("Could not enumerate subrepos for %s: %w", repoPath, err) + } + // A subrepoPath looks like e.g. /files/etc/yum.repos.d/almalinux-appstream.repo/appstream + for _, subrepoPath := range subrepoPaths { + keyPaths, err := handle.Aug_ls(subrepoPath) + if err != nil { + return fmt.Errorf("Could not enumerate keys for %s: %w", subrepoPath, err) + } + var ( + numComments int + commentedBaseurl string + mirrorlist string + uncommentedBaseurl string + ) + // A keyPath looks like e.g. /files/etc/yum.repos.d/almalinux-appstream.repo/appstream/mirrorlist + for _, keyPath := range keyPaths { + // extract the last part of the node path + key, err := handle.Aug_label(keyPath) + if err != nil { + return fmt.Errorf("Could not get label of %s: %w", keyPath, err) + } + value, err := handle.Aug_get(keyPath) + if err != nil { + return fmt.Errorf("Could not get %s: %w", keyPath, err) + } + if strings.HasPrefix(key, "#comment") { + numComments += 1 + if strings.HasPrefix(value, "baseurl=") { + commentedBaseurl = strings.Split(value, "=")[1] + } + } else if key == "mirrorlist" { + mirrorlist = value + } else if key == "baseurl" { + uncommentedBaseurl = value + } + } + var baseurl string + if uncommentedBaseurl != "" { + baseurl = transformBaseurl(uncommentedBaseurl) + if baseurl == uncommentedBaseurl { + baseurl = "" + } + } else if commentedBaseurl != "" { + baseurl = transformBaseurl(commentedBaseurl) + if baseurl == commentedBaseurl { + baseurl = "" + } + } + if baseurl != "" { + log.Debug().Msg(fmt.Sprintf("Setting %s to %s", subrepoPath+"/baseurl", baseurl)) + if err = handle.Aug_set(subrepoPath+"/baseurl", baseurl); err != nil { + return fmt.Errorf("Could not set baseurl for %s: %w", subrepoPath, err) + } + if mirrorlist != "" { + // comment out the mirrorlist line + log.Debug().Msg("Commenting out " + subrepoPath + "/mirrorlist") + if _, err = handle.Aug_rm(subrepoPath + "/mirrorlist"); err != nil { + return fmt.Errorf("Could not remove %s: %w", subrepoPath+"/mirrorlist", err) + } + if err = mgr.addAugeasComment(handle, numComments, subrepoPath, "mirrorlist="+mirrorlist); err != nil { + return fmt.Errorf("Could not insert comment in %s: %w", subrepoPath, err) + } + } + } + } + } + return +} + +// requires an Augeas handle to be open +func (mgr *TemplateManager) replaceDebianMirrorUrls(handle *guestfs.Guestfs) (err error) { + log := mgr.logger + // Some Augeas nodes under /files/etc/apt/sources.list are comments, + // so we use /*/uri to make sure that we only get the actual entries + sourcesListEntries, err := handle.Aug_match("/files/etc/apt/sources.list/*/uri") + if err != nil { + return + } + for _, uriPath := range sourcesListEntries { + var ( + uriValue string + parsedUrl *url.URL + distValue string + typeValue string + ) + if uriValue, err = handle.Aug_get(uriPath); err != nil { + return + } + parsedUrl, err = url.Parse(uriValue) + if err != nil { + return + } + parsedUrl.Host = mgr.cfg.MirrorHost + newUriValue := parsedUrl.String() + // strip off the "/uri" from the node path + entryPath := uriPath[:len(uriPath)-4] + typePath := entryPath + "/type" + if typeValue, err = handle.Aug_get(typePath); err != nil { + return + } + distPath := entryPath + "/distribution" + if distValue, err = handle.Aug_get(distPath); err != nil { + return + } + if typeValue == "deb-src" { + log.Debug(). + Str("URL", uriValue). + Str("distribution", distValue). + Msg("Removing deb-src entry") + if _, err = handle.Aug_rm(entryPath); err != nil { + return + } + continue + } + if uriValue == newUriValue { + continue + } + log.Debug(). + Str("distribution", distValue). + Str("oldURL", uriValue). + Str("newURL", newUriValue). + Msg("Replacing URL in sources.list") + if err = handle.Aug_set(uriPath, newUriValue); err != nil { + return + } + } + return +} + +// requires an Augeas handle to be open +func (mgr *TemplateManager) updateSshdConfig(handle *guestfs.Guestfs) error { + mgr.logger.Debug().Msg("Setting PrintLastLog=no in sshd_config") + return handle.Aug_set("/files/etc/ssh/sshd_config/PrintLastLog", "no") +} + +func (mgr *TemplateManager) setTimesyncdConf(handle *guestfs.Guestfs) (err error) { + mgr.logger.Debug().Msg("Writing custom timesyncd.conf") + if err = handle.Mkdir_p("/etc/systemd/timesyncd.conf.d"); err != nil { + return + } + return handle.Write("/etc/systemd/timesyncd.conf.d/csclub.conf", getResource("timesyncd.conf")) +} + +func (mgr *TemplateManager) setMotd(handle *guestfs.Guestfs) error { + mgr.logger.Debug().Msg("Writing to /etc/motd") + return handle.Write("/etc/motd", getResource("motd")) +} diff --git a/pkg/distros/ubuntu.go b/pkg/distros/ubuntu.go new file mode 100644 index 0000000..4fdb36f --- /dev/null +++ b/pkg/distros/ubuntu.go @@ -0,0 +1,132 @@ +package distros + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "github.com/rs/zerolog/log" + "libguestfs.org/guestfs" + + "git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" +) + +type UbuntuTemplateManager struct { + TemplateManager +} + +// NewUbuntuTemplateManager returns a UbuntuTemplateManager with the given +// config values. +func NewUbuntuTemplateManager(cfg *config.Config) *UbuntuTemplateManager { + logger := log.With().Str("distro", "ubuntu").Logger() + // The child embeds the parent, and the parent has a reference to the child + ubuntuTemplateManager := UbuntuTemplateManager{ + TemplateManager{ + cfg: cfg, + logger: &logger, + impl: nil, + }, + } + ubuntuTemplateManager.TemplateManager.impl = &ubuntuTemplateManager + return &ubuntuTemplateManager +} + +type UbuntuVersions struct { + Products map[string]struct { + Arch string `json:"arch"` + Release string `json:"release"` + ReleaseTitle string `json:"release_title"` + Version string `json:"version"` + } `json:"products"` +} + +// GetLatestVersion returns the version and codename of the latest Ubuntu release. +// e.g. version = 22.04, codename = jammy +func (mgr *UbuntuTemplateManager) GetLatestVersion() (version string, codename string, err error) { + resp, err := http.Get("https://cloud-images.ubuntu.com/releases/streams/v1/com.ubuntu.cloud:released:download.json") + if err != nil { + return + } + buf, err := io.ReadAll(resp.Body) + if err != nil { + return + } + var data UbuntuVersions + if err = json.Unmarshal(buf, &data); err != nil { + return + } + for _, val := range data.Products { + if val.Arch == "amd64" && strings.HasSuffix(val.ReleaseTitle, "LTS") && val.Version > version { + version = val.Version + codename = val.Release + } + } + if version == "" { + err = errors.New("Could not find latest LTS version of Ubuntu") + } + return +} + +// DownloadTemplate downloads the cloud image for the given Ubuntu codename, +// and returns the filename to which it was written. +func (mgr *UbuntuTemplateManager) DownloadTemplate(codename string) (path string, err error) { + filename := fmt.Sprintf("%s-server-cloudimg-amd64-disk-kvm.img", codename) + url := fmt.Sprintf("https://cloud-images.ubuntu.com/%s/current/%s", codename, filename) + return mgr.DownloadTemplateGeneric(filename, url) +} + +func (mgr *UbuntuTemplateManager) addCloudInitSnippet(handle *guestfs.Guestfs) error { + path := "/etc/cloud/cloud.cfg.d/99_csclub.cfg" + mgr.logger.Debug().Msg("Writing to " + path) + return handle.Write(path, getResource("ubuntu-cloud-init")) +} + +func (mgr *UbuntuTemplateManager) disableNoisyMotdMessages(handle *guestfs.Guestfs) { + log := mgr.logger + // Setting these files to non-executable (chmod a-x) disables them + filesToDisable := []string{ + "00-header", + "10-help-text", + "50-landscape-sysinfo", + "50-motd-news", + "88-esm-announce", + } + for _, filename := range filesToDisable { + path := "/etc/update-motd.d/" + filename + log.Debug().Msg("Disabling " + path) + if err := handle.Chmod(0644, path); err != nil { + log.Warn().Err(err).Msg("Could not disable " + path) + } + } +} + +func (mgr *UbuntuTemplateManager) HasSELinuxEnabled() bool { + return false +} + +func (mgr *UbuntuTemplateManager) GetIpv6ScriptsTemplateData() map[string]string { + return map[string]string{ + "networkService": "systemd-networkd.service", + "networkTarget": "network.target", + } +} + +func (mgr *UbuntuTemplateManager) PerformDistroSpecificModifications(handle *guestfs.Guestfs) (err error) { + if err = mgr.setTimesyncdConf(handle); err != nil { + return + } + if err = mgr.setDhclientOptions(handle); err != nil { + return + } + if err = mgr.replaceDebianMirrorUrls(handle); err != nil { + return + } + if err = mgr.addCloudInitSnippet(handle); err != nil { + return + } + mgr.disableNoisyMotdMessages(handle) + return +} diff --git a/scripts/download-deps.sh b/scripts/download-deps.sh new file mode 100755 index 0000000..cd35ef9 --- /dev/null +++ b/scripts/download-deps.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -e + +# This worked on Debian bullseye on the CSC machines - will need to +# be updated for other versions or other distros +# TODO: automatically generate this list from the core dependencies +# (golang-guestfs-dev, libvirt0, qemu-system-x86) +dependencies=( + golang-guestfs-dev + libbrlapi0 + libcacard0 + libcapstone4 + libdaxctl1 + libfdt1 + libguestfs0 + libguestfs-dev + libndctl6 + libpmem1 + libslirp0 + libspice-server1 + liburing1 + libusbredirparser1 + libvdeplug2 + libvirglrenderer1 + libvirt0 + libxencall1 + libxendevicemodel1 + libxenevtchn1 + libxenforeignmemory1 + libxengnttab1 + libxenmisc4 + libxenstore3 + libxentoolcore1 + libxentoollog1 + libyajl2 + qemu-system-x86 + qemu-utils + seabios +) + +if [ $# -eq 1 ] && [ "$1" = guestfish ]; then + dependencies=("${dependencies[@]}" libguestfs-tools libconfig9) +fi + +is_installed() { + status=$(dpkg-query --show --showformat '${db:Status-Status}' "$1" 2>/dev/null) + [ $? -eq 0 ] && [ "$status" = installed ] +} + +mkdir -p guestfs/deps +cd guestfs + +for dep in "${dependencies[@]}"; do + set -x + apt download $dep + dpkg -x ${dep}_*.deb deps + set +x +done +rm *.deb + +go mod init guestfs diff --git a/scripts/qemu.sh b/scripts/qemu.sh new file mode 100755 index 0000000..1df53e1 --- /dev/null +++ b/scripts/qemu.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +if [ -x /usr/bin/qemu-system-x86_64 ]; then + exec /usr/bin/qemu-system-x86_64 "$@" +fi +DEPS_DIR=guestfs/deps +exec $DEPS_DIR/usr/bin/qemu-system-x86_64 -L $DEPS_DIR/usr/share/seabios "$@"