first commit
This commit is contained in:
commit
efc45b65ce
|
|
@ -0,0 +1,14 @@
|
|||
module git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader
|
||||
|
||||
go 1.19
|
||||
|
||||
require (
|
||||
github.com/rs/zerolog v1.28.0
|
||||
golang.org/x/net v0.5.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
golang.org/x/sys v0.4.0 // indirect
|
||||
)
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
|
||||
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY=
|
||||
github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
|
||||
golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
|
||||
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
|
||||
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader/pkg/config"
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader/pkg/upgrader"
|
||||
)
|
||||
|
||||
func setupLogging() {
|
||||
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339})
|
||||
}
|
||||
|
||||
func main() {
|
||||
setupLogging()
|
||||
cfg := config.New()
|
||||
k8sUpgrader := upgrader.New(cfg)
|
||||
k8sUpgrader.DoUpgrade()
|
||||
}
|
||||
|
|
@ -0,0 +1,415 @@
|
|||
// Package cloudstack provides utilities for interacting with a CloudStack
|
||||
// management server.
|
||||
package cloudstack
|
||||
|
||||
/*
|
||||
* See https://cloudstack.apache.org/api/apidocs-4.16/ for API docs
|
||||
*/
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader/pkg/config"
|
||||
)
|
||||
|
||||
// A CloudstackClient interacts with a CloudStack management server via the
|
||||
// REST API.
|
||||
type CloudstackClient struct {
|
||||
cfg *config.Config
|
||||
}
|
||||
|
||||
// New returns a CloudStack client with the given config values.
|
||||
func New(cfg *config.Config) *CloudstackClient {
|
||||
return &CloudstackClient{cfg: cfg}
|
||||
}
|
||||
|
||||
// CloudStack uses java.net.URLEncoder when calculating signatures:
|
||||
// https://github.com/apache/cloudstack/blob/main/server/src/main/java/com/cloud/api/ApiServer.java
|
||||
// Unfortunately java.net.URLEncoder transforms "~" into "%7E".
|
||||
// This is a bug: https://bugs.openjdk.org/browse/JDK-8204530
|
||||
// So, we need to match Java's behaviour so that the server does not
|
||||
// reject our signature.
|
||||
// TODO: file a GitHub issue in the cloudmonkey repo
|
||||
func urlQueryEscape(value string) string {
|
||||
return strings.ReplaceAll(url.QueryEscape(value), "~", "%7E")
|
||||
}
|
||||
|
||||
// Adapted from https://github.com/apache/cloudstack-cloudmonkey/blob/main/cmd/network.go
|
||||
func encodeRequestParams(params map[string]string) string {
|
||||
if params == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(params))
|
||||
for key := range params {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var buf bytes.Buffer
|
||||
for _, key := range keys {
|
||||
value := params[key]
|
||||
if buf.Len() > 0 {
|
||||
buf.WriteByte('&')
|
||||
}
|
||||
buf.WriteString(key)
|
||||
buf.WriteString("=")
|
||||
buf.WriteString(urlQueryEscape(value))
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// See http://docs.cloudstack.apache.org/en/4.16.0.0/developersguide/dev.html#how-to-sign-an-api-call-with-python
|
||||
func (client *CloudstackClient) createURL(params map[string]string) string {
|
||||
cfg := client.cfg
|
||||
params["apiKey"] = cfg.CloudstackApiKey
|
||||
params["response"] = "json"
|
||||
|
||||
// adapted from https://github.com/apache/cloudstack-cloudmonkey/blob/main/cmd/network.go
|
||||
encodedParams := encodeRequestParams(params)
|
||||
mac := hmac.New(sha1.New, []byte(cfg.CloudstackSecretKey))
|
||||
mac.Write([]byte(strings.Replace(strings.ToLower(encodedParams), "+", "%20", -1)))
|
||||
signature := base64.StdEncoding.EncodeToString(mac.Sum(nil))
|
||||
encodedParams += fmt.Sprintf("&signature=%s", urlQueryEscape(signature))
|
||||
|
||||
return fmt.Sprintf("%s?%s", cfg.CloudstackApiBaseUrl, encodedParams)
|
||||
}
|
||||
|
||||
func getDeserializedResponse(url string, unmarshaledValue interface{}) {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
buf, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if err = json.Unmarshal(buf, unmarshaledValue); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
type ErrorInfo struct {
|
||||
ErrorCode int `json:"errorcode"`
|
||||
ErrorText string `json:"errortext"`
|
||||
}
|
||||
|
||||
func checkErrorInfo(data *ErrorInfo) {
|
||||
if data.ErrorCode != 0 {
|
||||
panic(fmt.Sprintf("Error %d: %s", data.ErrorCode, data.ErrorText))
|
||||
}
|
||||
}
|
||||
|
||||
// See https://cloudstack.apache.org/api/apidocs-4.16/apis/listKubernetesSupportedVersions.html
|
||||
type KubernetesSupportedVersionResponse struct {
|
||||
Id string `json:"id"`
|
||||
IsoId string `json:"isoid"`
|
||||
// e.g. "1.22.2-Kubernetes-Binaries-ISO"
|
||||
IsoName string `json:"isoname"`
|
||||
// e.g. "Ready"
|
||||
IsoState string `json:"isostate"`
|
||||
MinCpuNumber int `json:"mincpunumber"`
|
||||
MinMemory int `json:"minmemory"`
|
||||
Name string `json:"name"`
|
||||
// e.g. "1.22.2"
|
||||
SemanticVersion string `json:"semanticversion"`
|
||||
// e.g. "Enabled"
|
||||
State string `json:"state"`
|
||||
SupportsAutoscaling bool `json:"supportsautoscaling"`
|
||||
SupportsHA bool `json:"supportsha"`
|
||||
}
|
||||
|
||||
type ListKubernetesSupportedVersionsResponse struct {
|
||||
ErrorInfo
|
||||
Count int `json:"count"`
|
||||
KubernetesSupportedVersion []KubernetesSupportedVersionResponse `json:"kubernetessupportedversion"`
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) listKubernetesSupportedVersionsById(id *string) *ListKubernetesSupportedVersionsResponse {
|
||||
params := map[string]string{"command": "listKubernetesSupportedVersions"}
|
||||
if id != nil {
|
||||
params["id"] = *id
|
||||
}
|
||||
url := client.createURL(params)
|
||||
responseWrapper := struct {
|
||||
Response ListKubernetesSupportedVersionsResponse `json:"listkubernetessupportedversionsresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
return data
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) ListKubernetesSupportedVersions() *ListKubernetesSupportedVersionsResponse {
|
||||
return client.listKubernetesSupportedVersionsById(nil)
|
||||
}
|
||||
|
||||
type AddKubernetesSupportedVersionResponse struct {
|
||||
ErrorInfo
|
||||
KubernetesSupportedVersion KubernetesSupportedVersionResponse `json:"kubernetessupportedversion"`
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) AddKubernetesSupportedVersion(semver string, downloadURL string) *KubernetesSupportedVersionResponse {
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "addKubernetesSupportedVersion",
|
||||
"mincpunumber": "2",
|
||||
"minmemory": "2048",
|
||||
"semanticversion": semver,
|
||||
"name": semver,
|
||||
"url": downloadURL,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response AddKubernetesSupportedVersionResponse `json:"addkubernetessupportedversionresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
isoState := data.KubernetesSupportedVersion.IsoState
|
||||
isoID := data.KubernetesSupportedVersion.Id
|
||||
const maxTimeToWait = 10 * time.Minute
|
||||
const retryInterval = 20 * time.Second
|
||||
const maxTries = int(maxTimeToWait / retryInterval)
|
||||
for i := 0; i < maxTries; i++ {
|
||||
log.Debug().
|
||||
Str("id", isoID).
|
||||
Str("isoState", isoState).
|
||||
Msg("Waiting for ISO to become ready...")
|
||||
if isoState == "Ready" {
|
||||
break
|
||||
}
|
||||
time.Sleep(retryInterval)
|
||||
versionsById := client.listKubernetesSupportedVersionsById(&isoID)
|
||||
if versionsById.Count != 1 {
|
||||
panic(fmt.Sprintf("Expected to get 1 version for ISO, got %d", versionsById.Count))
|
||||
}
|
||||
isoState = versionsById.KubernetesSupportedVersion[0].IsoState
|
||||
}
|
||||
if isoState != "Ready" {
|
||||
panic("Timed out waiting for ISO to become ready")
|
||||
}
|
||||
return &data.KubernetesSupportedVersion
|
||||
}
|
||||
|
||||
type ListDomainsResponse struct {
|
||||
ErrorInfo
|
||||
Count int `json:"count"`
|
||||
Domain []struct {
|
||||
Name string `json:"name"`
|
||||
Id string `json:"id"`
|
||||
} `json:"domain"`
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) GetDomainId(domainName string) string {
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "listDomains",
|
||||
"details": "min",
|
||||
"name": domainName,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response ListDomainsResponse `json:"listdomainsresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
if data.Count != 1 {
|
||||
panic("there should be one domain found")
|
||||
}
|
||||
return data.Domain[0].Id
|
||||
}
|
||||
|
||||
// See https://cloudstack.apache.org/api/apidocs-4.16/apis/listKubernetesClusters.html
|
||||
// Fields in which we are not interested have been omitted
|
||||
type KubernetesClusterResponse struct {
|
||||
Id string `json:"id"`
|
||||
Account string `json:"account"`
|
||||
Domain string `json:"string"`
|
||||
DomainId string `json:"domainid"`
|
||||
KubernetesVersionId string `json:"kubernetesversionid"`
|
||||
KubernetesVersionName string `json:"kubernetesversionname"`
|
||||
Name string `json:"name"`
|
||||
ServiceOfferingId string `json:"serviceofferingid"`
|
||||
ServiceOfferingName string `json:"serviceofferingname"`
|
||||
State string `json:"state"`
|
||||
TemplateId string `json:"templateid"`
|
||||
}
|
||||
|
||||
type ListKubernetesClustersResponse struct {
|
||||
ErrorInfo
|
||||
Count int `json:"count"`
|
||||
KubernetesCluster []KubernetesClusterResponse `json:"kubernetescluster"`
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) listKubernetesClusters(name string, domainId string) *ListKubernetesClustersResponse {
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "listKubernetesClusters",
|
||||
"name": name,
|
||||
"domainid": domainId,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response ListKubernetesClustersResponse `json:"listkubernetesclustersresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
return data
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) GetKubernetesCluster(name string) *KubernetesClusterResponse {
|
||||
domainId := client.GetDomainId("ROOT")
|
||||
clusters := client.listKubernetesClusters(name, domainId)
|
||||
if clusters.Count != 1 {
|
||||
panic(fmt.Sprintf("Expected to find one k8s cluster, found %d instead", clusters.Count))
|
||||
}
|
||||
return &clusters.KubernetesCluster[0]
|
||||
}
|
||||
|
||||
type QueryAsyncJobResultResponse struct {
|
||||
ErrorInfo
|
||||
Created string `json:"created"`
|
||||
Completed string `json:"completed"`
|
||||
JobStatus int `json:"jobstatus"`
|
||||
JobResultCode int `json:"jobresultcode"`
|
||||
JobResult struct {
|
||||
// These fields are mutually exclusive
|
||||
ErrorText string `json:"errortext"`
|
||||
Success bool `json:"success"`
|
||||
} `json:"jobresult"`
|
||||
}
|
||||
|
||||
type JobFailedError struct {
|
||||
JobID string
|
||||
Status int
|
||||
ResultCode int
|
||||
ErrorText string
|
||||
}
|
||||
|
||||
func (e *JobFailedError) Error() string {
|
||||
return fmt.Sprintf("Job %s has status code %d", e.JobID, e.Status)
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) WaitForJobToComplete(jobID string) error {
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "queryAsyncJobResult",
|
||||
"jobid": jobID,
|
||||
})
|
||||
const maxTimeToWait = 1 * time.Hour
|
||||
const retryInterval = 15 * time.Second
|
||||
const maxTries = int(maxTimeToWait / retryInterval)
|
||||
for i := 0; i < maxTries; i++ {
|
||||
responseWrapper := struct {
|
||||
Response QueryAsyncJobResultResponse `json:"queryasyncjobresultresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
if data.JobStatus == 0 {
|
||||
log.Debug().Str("id", jobID).Msg("job is still pending")
|
||||
time.Sleep(retryInterval)
|
||||
continue
|
||||
} else if data.JobStatus == 1 {
|
||||
log.Debug().Str("id", jobID).Msg("job completed successfully")
|
||||
return nil
|
||||
} else {
|
||||
// either an error, or an unknown status code
|
||||
// see https://github.com/apache/cloudstack-cloudmonkey/blob/main/cmd/network.go
|
||||
return &JobFailedError{
|
||||
JobID: jobID,
|
||||
Status: data.JobStatus,
|
||||
ResultCode: data.JobResultCode,
|
||||
ErrorText: data.JobResult.ErrorText,
|
||||
}
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("Job %s took too long", jobID)
|
||||
}
|
||||
|
||||
type GenericAsyncResponse struct {
|
||||
ErrorInfo
|
||||
JobId string `json:"jobid"`
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) UpgradeKubernetesCluster(clusterID string, kubernetesVersionID string) error {
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "upgradeKubernetesCluster",
|
||||
"id": clusterID,
|
||||
"kubernetesversionid": kubernetesVersionID,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response GenericAsyncResponse `json:"upgradekubernetesclusterresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
return client.WaitForJobToComplete(data.JobId)
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) StopKubernetesCluster(clusterID string) {
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "stopKubernetesCluster",
|
||||
"id": clusterID,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response GenericAsyncResponse `json:"stopkubernetesclusterresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
err := client.WaitForJobToComplete(data.JobId)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) StartKubernetesCluster(clusterID string) {
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "startKubernetesCluster",
|
||||
"id": clusterID,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response GenericAsyncResponse `json:"startkubernetesclusterresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
err := client.WaitForJobToComplete(data.JobId)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
type DeleteKubernetesSupportedVersionResponse struct {
|
||||
ErrorInfo
|
||||
JobId string `json:"jobid"`
|
||||
}
|
||||
|
||||
func (client *CloudstackClient) DeleteKubernetesSupportedVersion(kubernetesVersionID string) error {
|
||||
url := client.createURL(map[string]string{
|
||||
"command": "deleteKubernetesSupportedVersion",
|
||||
"id": kubernetesVersionID,
|
||||
})
|
||||
responseWrapper := struct {
|
||||
Response DeleteKubernetesSupportedVersionResponse `json:"deletekubernetessupportedversionresponse"`
|
||||
}{}
|
||||
getDeserializedResponse(url, &responseWrapper)
|
||||
data := &responseWrapper.Response
|
||||
checkErrorInfo(&data.ErrorInfo)
|
||||
return client.WaitForJobToComplete(data.JobId)
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
package config
|
||||
|
||||
import "os"
|
||||
|
||||
// A Config holds all of the configuration values needed for the program.
|
||||
type Config struct {
|
||||
CloudstackApiKey string
|
||||
CloudstackSecretKey string
|
||||
CloudstackApiBaseUrl string
|
||||
ClusterName string
|
||||
EmailServer string
|
||||
EmailSender string
|
||||
EmailSenderName string
|
||||
EmailRecipient string
|
||||
EmailReplyTo string
|
||||
}
|
||||
|
||||
func New() *Config {
|
||||
cfg := &Config{
|
||||
CloudstackApiKey: os.Getenv("CLOUDSTACK_API_KEY"),
|
||||
CloudstackSecretKey: os.Getenv("CLOUDSTACK_SECRET_KEY"),
|
||||
ClusterName: os.Getenv("CLUSTER_NAME"),
|
||||
EmailRecipient: os.Getenv("EMAIL_RECIPIENT"),
|
||||
}
|
||||
if cfg.CloudstackApiKey == "" {
|
||||
panic("CLOUDSTACK_API_KEY is empty or not set")
|
||||
}
|
||||
if cfg.CloudstackSecretKey == "" {
|
||||
panic("CLOUDSTACK_SECRET_KEY is empty or not set")
|
||||
}
|
||||
if cfg.ClusterName == "" {
|
||||
panic("CLUSTER_NAME is empty or not set")
|
||||
}
|
||||
if cfg.EmailRecipient == "" {
|
||||
panic("EMAIL_RECIPIENT is empty or not set")
|
||||
}
|
||||
// These should never change
|
||||
cfg.CloudstackApiBaseUrl = "https://cloud.csclub.uwaterloo.ca/client/api"
|
||||
cfg.EmailServer = "mail.csclub.uwaterloo.ca:25"
|
||||
cfg.EmailSender = "cloudstack-k8s-upgrader@csclub.uwaterloo.ca"
|
||||
cfg.EmailSenderName = "cloudstack-k8s-upgrader"
|
||||
cfg.EmailReplyTo = "no-reply@csclub.uwaterloo.ca"
|
||||
return cfg
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
From: {{ .cfg.EmailSenderName }} <{{ .cfg.EmailSender }}>
|
||||
To: {{ .cfg.EmailRecipient }}
|
||||
Reply-To: {{ .cfg.EmailReplyTo }}
|
||||
Subject: New Kubernetes version: {{ .nextVerStr }}
|
||||
Date: {{ .date }}
|
||||
|
||||
Hello syscom,
|
||||
|
||||
This is an automated message from cloudstack-k8s-upgrader, the
|
||||
automated Kubernetes upgrader for CloudStack.
|
||||
|
||||
A new Kubernetes ISO, {{ .nextVerStr }}, has been uploaded to CloudStack.
|
||||
The managed CSC Kubernetes cluster has been upgraded to this version.
|
||||
If you have a chance, please test the cluster to make sure everything
|
||||
is working properly.
|
||||
|
||||
If you have any issues with cloudstack-k8s-upgrader, please report them here:
|
||||
https://git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader/issues
|
||||
|
||||
Sincerely,
|
||||
cloudstack-k8s-upgrader
|
||||
|
|
@ -0,0 +1,218 @@
|
|||
package upgrader
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
_ "embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/smtp"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"golang.org/x/net/html"
|
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader/pkg/cloudstack"
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader/pkg/config"
|
||||
)
|
||||
|
||||
//go:embed email-message.txt
|
||||
var emailMessageTemplate string
|
||||
|
||||
type Upgrader struct {
|
||||
cfg *config.Config
|
||||
client *cloudstack.CloudstackClient
|
||||
}
|
||||
|
||||
func New(cfg *config.Config) *Upgrader {
|
||||
return &Upgrader{
|
||||
cfg: cfg,
|
||||
client: cloudstack.New(cfg),
|
||||
}
|
||||
}
|
||||
|
||||
type semanticVersion struct {
|
||||
major, minor, patch int
|
||||
}
|
||||
|
||||
func getSemanticVersion(s string) *semanticVersion {
|
||||
semver := semanticVersion{}
|
||||
n, err := fmt.Sscanf(s, "%d.%d.%d", &semver.major, &semver.minor, &semver.patch)
|
||||
if n != 3 || err != nil {
|
||||
panic(fmt.Sprintf("Invalid semantic version string: %s", s))
|
||||
}
|
||||
return &semver
|
||||
}
|
||||
|
||||
func (ver *semanticVersion) toString() string {
|
||||
return fmt.Sprintf("%d.%d.%d", ver.major, ver.minor, ver.patch)
|
||||
}
|
||||
|
||||
const ISO_DOWNLOAD_BASE_URL = "https://download.cloudstack.org/cks/"
|
||||
|
||||
func isHyperlink(node *html.Node) bool {
|
||||
return node.Type == html.ElementNode && node.Data == "a" &&
|
||||
node.FirstChild != nil && node.FirstChild.Type == html.TextNode
|
||||
}
|
||||
|
||||
func getNextUpgradableVersionFromHtmlNode(currentVer *semanticVersion, n *html.Node) *semanticVersion {
|
||||
if isHyperlink(n) {
|
||||
nextVer := semanticVersion{}
|
||||
n, err := fmt.Sscanf(
|
||||
n.FirstChild.Data, "setup-%d.%d.%d.iso",
|
||||
&nextVer.major, &nextVer.minor, &nextVer.patch,
|
||||
)
|
||||
if n == 3 && err == nil && nextVer.major >= currentVer.major {
|
||||
if nextVer.major > currentVer.major {
|
||||
panic("A new major version was detected. This kind of upgrade " +
|
||||
"is too dangerous to perform automatically.")
|
||||
}
|
||||
if nextVer.minor > currentVer.minor && nextVer.minor-currentVer.minor == 1 {
|
||||
return &nextVer
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
nextVer := getNextUpgradableVersionFromHtmlNode(currentVer, c)
|
||||
if nextVer != nil {
|
||||
return nextVer
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getNextUpgradableVersion(currentVer *semanticVersion) *semanticVersion {
|
||||
resp, err := http.Get(ISO_DOWNLOAD_BASE_URL)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
doc, err := html.Parse(resp.Body)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return getNextUpgradableVersionFromHtmlNode(currentVer, doc)
|
||||
}
|
||||
|
||||
func semverCompare(ver1, ver2 *semanticVersion) int {
|
||||
if ver1.major > ver2.major {
|
||||
return 1
|
||||
}
|
||||
if ver1.major < ver2.major {
|
||||
return -1
|
||||
}
|
||||
if ver1.minor > ver2.minor {
|
||||
return 1
|
||||
}
|
||||
if ver1.minor < ver2.minor {
|
||||
return -1
|
||||
}
|
||||
if ver1.patch > ver2.patch {
|
||||
return 1
|
||||
}
|
||||
if ver1.patch < ver2.patch {
|
||||
return -1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func getLatestExistingISO(versions []cloudstack.KubernetesSupportedVersionResponse) *cloudstack.KubernetesSupportedVersionResponse {
|
||||
latestSoFar := &versions[0]
|
||||
latestVer := getSemanticVersion(latestSoFar.SemanticVersion)
|
||||
for i, _ := range versions {
|
||||
current := &versions[i]
|
||||
currentVer := getSemanticVersion(current.SemanticVersion)
|
||||
if semverCompare(currentVer, latestVer) > 0 {
|
||||
latestSoFar = current
|
||||
latestVer = currentVer
|
||||
}
|
||||
}
|
||||
return latestSoFar
|
||||
}
|
||||
|
||||
func (u *Upgrader) DoUpgrade() {
|
||||
cluster := u.client.GetKubernetesCluster(u.cfg.ClusterName)
|
||||
currentVersions := u.client.ListKubernetesSupportedVersions()
|
||||
if currentVersions.Count == 0 {
|
||||
panic("No k8s ISOs present")
|
||||
}
|
||||
currentISO := getLatestExistingISO(currentVersions.KubernetesSupportedVersion)
|
||||
currentVerStr := currentISO.SemanticVersion
|
||||
currentVer := getSemanticVersion(currentVerStr)
|
||||
nextVer := getNextUpgradableVersion(currentVer)
|
||||
if nextVer == nil {
|
||||
return
|
||||
}
|
||||
nextVerStr := nextVer.toString()
|
||||
log.Info().
|
||||
Str("currentVer", currentVerStr).
|
||||
Str("nextVer", nextVerStr).
|
||||
Msg("New version available")
|
||||
downloadURL := ISO_DOWNLOAD_BASE_URL + "setup-" + nextVerStr + ".iso"
|
||||
|
||||
log.Info().Msg("Adding new k8s ISO")
|
||||
newISO := u.client.AddKubernetesSupportedVersion(nextVerStr, downloadURL)
|
||||
|
||||
log.Info().Msg("Upgrading k8s cluster")
|
||||
err := u.client.UpgradeKubernetesCluster(cluster.Id, newISO.Id)
|
||||
var jobFailedError *cloudstack.JobFailedError
|
||||
if errors.As(err, &jobFailedError) &&
|
||||
strings.Contains(jobFailedError.ErrorText, "unable to upgrade Kubernetes node on VM") {
|
||||
// The upgrade might fail the first time.
|
||||
// Reboot the cluster and try again.
|
||||
log.Warn().
|
||||
Str("jobID", jobFailedError.JobID).
|
||||
Int("resultCode", jobFailedError.ResultCode).
|
||||
Msg("Job failed")
|
||||
log.Warn().Msg("Stopping cluster")
|
||||
u.client.StopKubernetesCluster(cluster.Id)
|
||||
log.Warn().Msg("Starting cluster")
|
||||
u.client.StartKubernetesCluster(cluster.Id)
|
||||
log.Debug().Msg("Waiting for cluster to stabilize after reboot")
|
||||
time.Sleep(5 * time.Minute)
|
||||
log.Info().Msg("Upgrading k8s cluster (2nd attempt)")
|
||||
err = u.client.UpgradeKubernetesCluster(cluster.Id, newISO.Id)
|
||||
}
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
log.Info().Msg("Deleting old k8s ISO")
|
||||
err = u.client.DeleteKubernetesSupportedVersion(currentISO.Id)
|
||||
if err != nil {
|
||||
log.Warn().Msg("Could not delete old k8s ISO: " + err.Error())
|
||||
}
|
||||
u.sendEmailNotification(nextVerStr)
|
||||
}
|
||||
|
||||
func (u *Upgrader) sendEmailNotification(nextVerStr string) {
|
||||
tmpl := template.Must(template.New("email-message").Parse(emailMessageTemplate))
|
||||
data := map[string]interface{}{
|
||||
"cfg": u.cfg,
|
||||
"date": time.Now().Format(time.RFC1123Z),
|
||||
"nextVerStr": nextVerStr,
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := tmpl.Execute(&buf, data); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// The lines of the body need to be CRLF terminated
|
||||
// See https://pkg.go.dev/net/smtp#SendMail
|
||||
// Make sure that email-message.txt uses Unix-style LF endings
|
||||
msg := bytes.ReplaceAll(buf.Bytes(), []byte("\n"), []byte("\r\n"))
|
||||
log.Debug().
|
||||
Str("to", u.cfg.EmailRecipient).
|
||||
Msg("sending email notification")
|
||||
if err := smtp.SendMail(
|
||||
u.cfg.EmailServer,
|
||||
nil, /* auth */
|
||||
u.cfg.EmailSender,
|
||||
[]string{u.cfg.EmailRecipient},
|
||||
msg,
|
||||
); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue