commit efc45b65ce165b8e32fc383cbd0e36f83982834d Author: Max Erenberg Date: Sat Jan 7 02:27:06 2023 -0500 first commit diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..3375957 --- /dev/null +++ b/go.mod @@ -0,0 +1,14 @@ +module git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader + +go 1.19 + +require ( + github.com/rs/zerolog v1.28.0 + golang.org/x/net v0.5.0 +) + +require ( + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + golang.org/x/sys v0.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d0b8e40 --- /dev/null +++ b/go.sum @@ -0,0 +1,16 @@ +github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= +github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/main.go b/main.go new file mode 100644 index 0000000..76d214e --- /dev/null +++ b/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "os" + "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + + "git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader/pkg/config" + "git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader/pkg/upgrader" +) + +func setupLogging() { + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}) +} + +func main() { + setupLogging() + cfg := config.New() + k8sUpgrader := upgrader.New(cfg) + k8sUpgrader.DoUpgrade() +} diff --git a/pkg/cloudstack/cloudstack.go b/pkg/cloudstack/cloudstack.go new file mode 100644 index 0000000..92145c2 --- /dev/null +++ b/pkg/cloudstack/cloudstack.go @@ -0,0 +1,415 @@ +// Package cloudstack provides utilities for interacting with a CloudStack +// management server. +package cloudstack + +/* + * See https://cloudstack.apache.org/api/apidocs-4.16/ for API docs + */ + +import ( + "bytes" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "github.com/rs/zerolog/log" + + "git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader/pkg/config" +) + +// A CloudstackClient interacts with a CloudStack management server via the +// REST API. +type CloudstackClient struct { + cfg *config.Config +} + +// New returns a CloudStack client with the given config values. +func New(cfg *config.Config) *CloudstackClient { + return &CloudstackClient{cfg: cfg} +} + +// CloudStack uses java.net.URLEncoder when calculating signatures: +// https://github.com/apache/cloudstack/blob/main/server/src/main/java/com/cloud/api/ApiServer.java +// Unfortunately java.net.URLEncoder transforms "~" into "%7E". +// This is a bug: https://bugs.openjdk.org/browse/JDK-8204530 +// So, we need to match Java's behaviour so that the server does not +// reject our signature. +// TODO: file a GitHub issue in the cloudmonkey repo +func urlQueryEscape(value string) string { + return strings.ReplaceAll(url.QueryEscape(value), "~", "%7E") +} + +// Adapted from https://github.com/apache/cloudstack-cloudmonkey/blob/main/cmd/network.go +func encodeRequestParams(params map[string]string) string { + if params == nil { + return "" + } + + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sort.Strings(keys) + + var buf bytes.Buffer + for _, key := range keys { + value := params[key] + if buf.Len() > 0 { + buf.WriteByte('&') + } + buf.WriteString(key) + buf.WriteString("=") + buf.WriteString(urlQueryEscape(value)) + } + return buf.String() +} + +// See http://docs.cloudstack.apache.org/en/4.16.0.0/developersguide/dev.html#how-to-sign-an-api-call-with-python +func (client *CloudstackClient) createURL(params map[string]string) string { + cfg := client.cfg + params["apiKey"] = cfg.CloudstackApiKey + params["response"] = "json" + + // adapted from https://github.com/apache/cloudstack-cloudmonkey/blob/main/cmd/network.go + encodedParams := encodeRequestParams(params) + mac := hmac.New(sha1.New, []byte(cfg.CloudstackSecretKey)) + mac.Write([]byte(strings.Replace(strings.ToLower(encodedParams), "+", "%20", -1))) + signature := base64.StdEncoding.EncodeToString(mac.Sum(nil)) + encodedParams += fmt.Sprintf("&signature=%s", urlQueryEscape(signature)) + + return fmt.Sprintf("%s?%s", cfg.CloudstackApiBaseUrl, encodedParams) +} + +func getDeserializedResponse(url string, unmarshaledValue interface{}) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + panic(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + buf, err := io.ReadAll(resp.Body) + if err != nil { + panic(err) + } + if err = json.Unmarshal(buf, unmarshaledValue); err != nil { + panic(err) + } +} + +type ErrorInfo struct { + ErrorCode int `json:"errorcode"` + ErrorText string `json:"errortext"` +} + +func checkErrorInfo(data *ErrorInfo) { + if data.ErrorCode != 0 { + panic(fmt.Sprintf("Error %d: %s", data.ErrorCode, data.ErrorText)) + } +} + +// See https://cloudstack.apache.org/api/apidocs-4.16/apis/listKubernetesSupportedVersions.html +type KubernetesSupportedVersionResponse struct { + Id string `json:"id"` + IsoId string `json:"isoid"` + // e.g. "1.22.2-Kubernetes-Binaries-ISO" + IsoName string `json:"isoname"` + // e.g. "Ready" + IsoState string `json:"isostate"` + MinCpuNumber int `json:"mincpunumber"` + MinMemory int `json:"minmemory"` + Name string `json:"name"` + // e.g. "1.22.2" + SemanticVersion string `json:"semanticversion"` + // e.g. "Enabled" + State string `json:"state"` + SupportsAutoscaling bool `json:"supportsautoscaling"` + SupportsHA bool `json:"supportsha"` +} + +type ListKubernetesSupportedVersionsResponse struct { + ErrorInfo + Count int `json:"count"` + KubernetesSupportedVersion []KubernetesSupportedVersionResponse `json:"kubernetessupportedversion"` +} + +func (client *CloudstackClient) listKubernetesSupportedVersionsById(id *string) *ListKubernetesSupportedVersionsResponse { + params := map[string]string{"command": "listKubernetesSupportedVersions"} + if id != nil { + params["id"] = *id + } + url := client.createURL(params) + responseWrapper := struct { + Response ListKubernetesSupportedVersionsResponse `json:"listkubernetessupportedversionsresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + return data +} + +func (client *CloudstackClient) ListKubernetesSupportedVersions() *ListKubernetesSupportedVersionsResponse { + return client.listKubernetesSupportedVersionsById(nil) +} + +type AddKubernetesSupportedVersionResponse struct { + ErrorInfo + KubernetesSupportedVersion KubernetesSupportedVersionResponse `json:"kubernetessupportedversion"` +} + +func (client *CloudstackClient) AddKubernetesSupportedVersion(semver string, downloadURL string) *KubernetesSupportedVersionResponse { + url := client.createURL(map[string]string{ + "command": "addKubernetesSupportedVersion", + "mincpunumber": "2", + "minmemory": "2048", + "semanticversion": semver, + "name": semver, + "url": downloadURL, + }) + responseWrapper := struct { + Response AddKubernetesSupportedVersionResponse `json:"addkubernetessupportedversionresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + isoState := data.KubernetesSupportedVersion.IsoState + isoID := data.KubernetesSupportedVersion.Id + const maxTimeToWait = 10 * time.Minute + const retryInterval = 20 * time.Second + const maxTries = int(maxTimeToWait / retryInterval) + for i := 0; i < maxTries; i++ { + log.Debug(). + Str("id", isoID). + Str("isoState", isoState). + Msg("Waiting for ISO to become ready...") + if isoState == "Ready" { + break + } + time.Sleep(retryInterval) + versionsById := client.listKubernetesSupportedVersionsById(&isoID) + if versionsById.Count != 1 { + panic(fmt.Sprintf("Expected to get 1 version for ISO, got %d", versionsById.Count)) + } + isoState = versionsById.KubernetesSupportedVersion[0].IsoState + } + if isoState != "Ready" { + panic("Timed out waiting for ISO to become ready") + } + return &data.KubernetesSupportedVersion +} + +type ListDomainsResponse struct { + ErrorInfo + Count int `json:"count"` + Domain []struct { + Name string `json:"name"` + Id string `json:"id"` + } `json:"domain"` +} + +func (client *CloudstackClient) GetDomainId(domainName string) string { + url := client.createURL(map[string]string{ + "command": "listDomains", + "details": "min", + "name": domainName, + }) + responseWrapper := struct { + Response ListDomainsResponse `json:"listdomainsresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + if data.Count != 1 { + panic("there should be one domain found") + } + return data.Domain[0].Id +} + +// See https://cloudstack.apache.org/api/apidocs-4.16/apis/listKubernetesClusters.html +// Fields in which we are not interested have been omitted +type KubernetesClusterResponse struct { + Id string `json:"id"` + Account string `json:"account"` + Domain string `json:"string"` + DomainId string `json:"domainid"` + KubernetesVersionId string `json:"kubernetesversionid"` + KubernetesVersionName string `json:"kubernetesversionname"` + Name string `json:"name"` + ServiceOfferingId string `json:"serviceofferingid"` + ServiceOfferingName string `json:"serviceofferingname"` + State string `json:"state"` + TemplateId string `json:"templateid"` +} + +type ListKubernetesClustersResponse struct { + ErrorInfo + Count int `json:"count"` + KubernetesCluster []KubernetesClusterResponse `json:"kubernetescluster"` +} + +func (client *CloudstackClient) listKubernetesClusters(name string, domainId string) *ListKubernetesClustersResponse { + url := client.createURL(map[string]string{ + "command": "listKubernetesClusters", + "name": name, + "domainid": domainId, + }) + responseWrapper := struct { + Response ListKubernetesClustersResponse `json:"listkubernetesclustersresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + return data +} + +func (client *CloudstackClient) GetKubernetesCluster(name string) *KubernetesClusterResponse { + domainId := client.GetDomainId("ROOT") + clusters := client.listKubernetesClusters(name, domainId) + if clusters.Count != 1 { + panic(fmt.Sprintf("Expected to find one k8s cluster, found %d instead", clusters.Count)) + } + return &clusters.KubernetesCluster[0] +} + +type QueryAsyncJobResultResponse struct { + ErrorInfo + Created string `json:"created"` + Completed string `json:"completed"` + JobStatus int `json:"jobstatus"` + JobResultCode int `json:"jobresultcode"` + JobResult struct { + // These fields are mutually exclusive + ErrorText string `json:"errortext"` + Success bool `json:"success"` + } `json:"jobresult"` +} + +type JobFailedError struct { + JobID string + Status int + ResultCode int + ErrorText string +} + +func (e *JobFailedError) Error() string { + return fmt.Sprintf("Job %s has status code %d", e.JobID, e.Status) +} + +func (client *CloudstackClient) WaitForJobToComplete(jobID string) error { + url := client.createURL(map[string]string{ + "command": "queryAsyncJobResult", + "jobid": jobID, + }) + const maxTimeToWait = 1 * time.Hour + const retryInterval = 15 * time.Second + const maxTries = int(maxTimeToWait / retryInterval) + for i := 0; i < maxTries; i++ { + responseWrapper := struct { + Response QueryAsyncJobResultResponse `json:"queryasyncjobresultresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + if data.JobStatus == 0 { + log.Debug().Str("id", jobID).Msg("job is still pending") + time.Sleep(retryInterval) + continue + } else if data.JobStatus == 1 { + log.Debug().Str("id", jobID).Msg("job completed successfully") + return nil + } else { + // either an error, or an unknown status code + // see https://github.com/apache/cloudstack-cloudmonkey/blob/main/cmd/network.go + return &JobFailedError{ + JobID: jobID, + Status: data.JobStatus, + ResultCode: data.JobResultCode, + ErrorText: data.JobResult.ErrorText, + } + } + } + return fmt.Errorf("Job %s took too long", jobID) +} + +type GenericAsyncResponse struct { + ErrorInfo + JobId string `json:"jobid"` +} + +func (client *CloudstackClient) UpgradeKubernetesCluster(clusterID string, kubernetesVersionID string) error { + url := client.createURL(map[string]string{ + "command": "upgradeKubernetesCluster", + "id": clusterID, + "kubernetesversionid": kubernetesVersionID, + }) + responseWrapper := struct { + Response GenericAsyncResponse `json:"upgradekubernetesclusterresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + return client.WaitForJobToComplete(data.JobId) +} + +func (client *CloudstackClient) StopKubernetesCluster(clusterID string) { + url := client.createURL(map[string]string{ + "command": "stopKubernetesCluster", + "id": clusterID, + }) + responseWrapper := struct { + Response GenericAsyncResponse `json:"stopkubernetesclusterresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + err := client.WaitForJobToComplete(data.JobId) + if err != nil { + panic(err) + } +} + +func (client *CloudstackClient) StartKubernetesCluster(clusterID string) { + url := client.createURL(map[string]string{ + "command": "startKubernetesCluster", + "id": clusterID, + }) + responseWrapper := struct { + Response GenericAsyncResponse `json:"startkubernetesclusterresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + err := client.WaitForJobToComplete(data.JobId) + if err != nil { + panic(err) + } +} + +type DeleteKubernetesSupportedVersionResponse struct { + ErrorInfo + JobId string `json:"jobid"` +} + +func (client *CloudstackClient) DeleteKubernetesSupportedVersion(kubernetesVersionID string) error { + url := client.createURL(map[string]string{ + "command": "deleteKubernetesSupportedVersion", + "id": kubernetesVersionID, + }) + responseWrapper := struct { + Response DeleteKubernetesSupportedVersionResponse `json:"deletekubernetessupportedversionresponse"` + }{} + getDeserializedResponse(url, &responseWrapper) + data := &responseWrapper.Response + checkErrorInfo(&data.ErrorInfo) + return client.WaitForJobToComplete(data.JobId) +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..846ddd6 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,44 @@ +package config + +import "os" + +// A Config holds all of the configuration values needed for the program. +type Config struct { + CloudstackApiKey string + CloudstackSecretKey string + CloudstackApiBaseUrl string + ClusterName string + EmailServer string + EmailSender string + EmailSenderName string + EmailRecipient string + EmailReplyTo string +} + +func New() *Config { + cfg := &Config{ + CloudstackApiKey: os.Getenv("CLOUDSTACK_API_KEY"), + CloudstackSecretKey: os.Getenv("CLOUDSTACK_SECRET_KEY"), + ClusterName: os.Getenv("CLUSTER_NAME"), + EmailRecipient: os.Getenv("EMAIL_RECIPIENT"), + } + if cfg.CloudstackApiKey == "" { + panic("CLOUDSTACK_API_KEY is empty or not set") + } + if cfg.CloudstackSecretKey == "" { + panic("CLOUDSTACK_SECRET_KEY is empty or not set") + } + if cfg.ClusterName == "" { + panic("CLUSTER_NAME is empty or not set") + } + if cfg.EmailRecipient == "" { + panic("EMAIL_RECIPIENT is empty or not set") + } + // These should never change + cfg.CloudstackApiBaseUrl = "https://cloud.csclub.uwaterloo.ca/client/api" + cfg.EmailServer = "mail.csclub.uwaterloo.ca:25" + cfg.EmailSender = "cloudstack-k8s-upgrader@csclub.uwaterloo.ca" + cfg.EmailSenderName = "cloudstack-k8s-upgrader" + cfg.EmailReplyTo = "no-reply@csclub.uwaterloo.ca" + return cfg +} diff --git a/pkg/upgrader/email-message.txt b/pkg/upgrader/email-message.txt new file mode 100644 index 0000000..41bf40e --- /dev/null +++ b/pkg/upgrader/email-message.txt @@ -0,0 +1,21 @@ +From: {{ .cfg.EmailSenderName }} <{{ .cfg.EmailSender }}> +To: {{ .cfg.EmailRecipient }} +Reply-To: {{ .cfg.EmailReplyTo }} +Subject: New Kubernetes version: {{ .nextVerStr }} +Date: {{ .date }} + +Hello syscom, + +This is an automated message from cloudstack-k8s-upgrader, the +automated Kubernetes upgrader for CloudStack. + +A new Kubernetes ISO, {{ .nextVerStr }}, has been uploaded to CloudStack. +The managed CSC Kubernetes cluster has been upgraded to this version. +If you have a chance, please test the cluster to make sure everything +is working properly. + +If you have any issues with cloudstack-k8s-upgrader, please report them here: +https://git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader/issues + +Sincerely, +cloudstack-k8s-upgrader diff --git a/pkg/upgrader/upgrader.go b/pkg/upgrader/upgrader.go new file mode 100644 index 0000000..951a792 --- /dev/null +++ b/pkg/upgrader/upgrader.go @@ -0,0 +1,218 @@ +package upgrader + +import ( + "bytes" + _ "embed" + "errors" + "fmt" + "net/http" + "net/smtp" + "strings" + "text/template" + "time" + + "github.com/rs/zerolog/log" + "golang.org/x/net/html" + + "git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader/pkg/cloudstack" + "git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader/pkg/config" +) + +//go:embed email-message.txt +var emailMessageTemplate string + +type Upgrader struct { + cfg *config.Config + client *cloudstack.CloudstackClient +} + +func New(cfg *config.Config) *Upgrader { + return &Upgrader{ + cfg: cfg, + client: cloudstack.New(cfg), + } +} + +type semanticVersion struct { + major, minor, patch int +} + +func getSemanticVersion(s string) *semanticVersion { + semver := semanticVersion{} + n, err := fmt.Sscanf(s, "%d.%d.%d", &semver.major, &semver.minor, &semver.patch) + if n != 3 || err != nil { + panic(fmt.Sprintf("Invalid semantic version string: %s", s)) + } + return &semver +} + +func (ver *semanticVersion) toString() string { + return fmt.Sprintf("%d.%d.%d", ver.major, ver.minor, ver.patch) +} + +const ISO_DOWNLOAD_BASE_URL = "https://download.cloudstack.org/cks/" + +func isHyperlink(node *html.Node) bool { + return node.Type == html.ElementNode && node.Data == "a" && + node.FirstChild != nil && node.FirstChild.Type == html.TextNode +} + +func getNextUpgradableVersionFromHtmlNode(currentVer *semanticVersion, n *html.Node) *semanticVersion { + if isHyperlink(n) { + nextVer := semanticVersion{} + n, err := fmt.Sscanf( + n.FirstChild.Data, "setup-%d.%d.%d.iso", + &nextVer.major, &nextVer.minor, &nextVer.patch, + ) + if n == 3 && err == nil && nextVer.major >= currentVer.major { + if nextVer.major > currentVer.major { + panic("A new major version was detected. This kind of upgrade " + + "is too dangerous to perform automatically.") + } + if nextVer.minor > currentVer.minor && nextVer.minor-currentVer.minor == 1 { + return &nextVer + } + } + return nil + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + nextVer := getNextUpgradableVersionFromHtmlNode(currentVer, c) + if nextVer != nil { + return nextVer + } + } + return nil +} + +func getNextUpgradableVersion(currentVer *semanticVersion) *semanticVersion { + resp, err := http.Get(ISO_DOWNLOAD_BASE_URL) + if err != nil { + panic(err) + } + defer resp.Body.Close() + doc, err := html.Parse(resp.Body) + if err != nil { + panic(err) + } + return getNextUpgradableVersionFromHtmlNode(currentVer, doc) +} + +func semverCompare(ver1, ver2 *semanticVersion) int { + if ver1.major > ver2.major { + return 1 + } + if ver1.major < ver2.major { + return -1 + } + if ver1.minor > ver2.minor { + return 1 + } + if ver1.minor < ver2.minor { + return -1 + } + if ver1.patch > ver2.patch { + return 1 + } + if ver1.patch < ver2.patch { + return -1 + } + return 0 +} + +func getLatestExistingISO(versions []cloudstack.KubernetesSupportedVersionResponse) *cloudstack.KubernetesSupportedVersionResponse { + latestSoFar := &versions[0] + latestVer := getSemanticVersion(latestSoFar.SemanticVersion) + for i, _ := range versions { + current := &versions[i] + currentVer := getSemanticVersion(current.SemanticVersion) + if semverCompare(currentVer, latestVer) > 0 { + latestSoFar = current + latestVer = currentVer + } + } + return latestSoFar +} + +func (u *Upgrader) DoUpgrade() { + cluster := u.client.GetKubernetesCluster(u.cfg.ClusterName) + currentVersions := u.client.ListKubernetesSupportedVersions() + if currentVersions.Count == 0 { + panic("No k8s ISOs present") + } + currentISO := getLatestExistingISO(currentVersions.KubernetesSupportedVersion) + currentVerStr := currentISO.SemanticVersion + currentVer := getSemanticVersion(currentVerStr) + nextVer := getNextUpgradableVersion(currentVer) + if nextVer == nil { + return + } + nextVerStr := nextVer.toString() + log.Info(). + Str("currentVer", currentVerStr). + Str("nextVer", nextVerStr). + Msg("New version available") + downloadURL := ISO_DOWNLOAD_BASE_URL + "setup-" + nextVerStr + ".iso" + + log.Info().Msg("Adding new k8s ISO") + newISO := u.client.AddKubernetesSupportedVersion(nextVerStr, downloadURL) + + log.Info().Msg("Upgrading k8s cluster") + err := u.client.UpgradeKubernetesCluster(cluster.Id, newISO.Id) + var jobFailedError *cloudstack.JobFailedError + if errors.As(err, &jobFailedError) && + strings.Contains(jobFailedError.ErrorText, "unable to upgrade Kubernetes node on VM") { + // The upgrade might fail the first time. + // Reboot the cluster and try again. + log.Warn(). + Str("jobID", jobFailedError.JobID). + Int("resultCode", jobFailedError.ResultCode). + Msg("Job failed") + log.Warn().Msg("Stopping cluster") + u.client.StopKubernetesCluster(cluster.Id) + log.Warn().Msg("Starting cluster") + u.client.StartKubernetesCluster(cluster.Id) + log.Debug().Msg("Waiting for cluster to stabilize after reboot") + time.Sleep(5 * time.Minute) + log.Info().Msg("Upgrading k8s cluster (2nd attempt)") + err = u.client.UpgradeKubernetesCluster(cluster.Id, newISO.Id) + } + if err != nil { + panic(err) + } + + log.Info().Msg("Deleting old k8s ISO") + err = u.client.DeleteKubernetesSupportedVersion(currentISO.Id) + if err != nil { + log.Warn().Msg("Could not delete old k8s ISO: " + err.Error()) + } + u.sendEmailNotification(nextVerStr) +} + +func (u *Upgrader) sendEmailNotification(nextVerStr string) { + tmpl := template.Must(template.New("email-message").Parse(emailMessageTemplate)) + data := map[string]interface{}{ + "cfg": u.cfg, + "date": time.Now().Format(time.RFC1123Z), + "nextVerStr": nextVerStr, + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + panic(err) + } + // The lines of the body need to be CRLF terminated + // See https://pkg.go.dev/net/smtp#SendMail + // Make sure that email-message.txt uses Unix-style LF endings + msg := bytes.ReplaceAll(buf.Bytes(), []byte("\n"), []byte("\r\n")) + log.Debug(). + Str("to", u.cfg.EmailRecipient). + Msg("sending email notification") + if err := smtp.SendMail( + u.cfg.EmailServer, + nil, /* auth */ + u.cfg.EmailSender, + []string{u.cfg.EmailRecipient}, + msg, + ); err != nil { + panic(err) + } +}