edit kube-controller-manager manifest after upgrade

.gitignore

@@ -0,0 +1 @@
+/cloudstack-k8s-upgrader

go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/rs/zerolog v1.28.0
 	golang.org/x/net v0.5.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (

go.sum

@@ -14,3 +14,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/cloudstack/cloudstack.go

@@ -242,6 +242,7 @@ type KubernetesClusterResponse struct {
 	Account               string `json:"account"`
 	Domain                string `json:"string"`
 	DomainId              string `json:"domainid"`
+	Endpoint              string `json:"endpoint"`
 	KubernetesVersionId   string `json:"kubernetesversionid"`
 	KubernetesVersionName string `json:"kubernetesversionname"`
 	Name                  string `json:"name"`

pkg/config/config.go

@@ -8,6 +8,7 @@ type Config struct {
 	CloudstackSecretKey  string
 	CloudstackApiBaseUrl string
 	ClusterName          string
+	SSHKeyPath           string
 	EmailServer          string
 	EmailSender          string
 	EmailSenderName      string
@@ -20,6 +21,7 @@ func New() *Config {
 		CloudstackApiKey:    os.Getenv("CLOUDSTACK_API_KEY"),
 		CloudstackSecretKey: os.Getenv("CLOUDSTACK_SECRET_KEY"),
 		ClusterName:         os.Getenv("CLUSTER_NAME"),
+		SSHKeyPath:          os.Getenv("SSH_KEY_PATH"),
 		EmailRecipient:      os.Getenv("EMAIL_RECIPIENT"),
 	}
 	if cfg.CloudstackApiKey == "" {
@@ -31,6 +33,9 @@ func New() *Config {
 	if cfg.ClusterName == "" {
 		panic("CLUSTER_NAME is empty or not set")
 	}
+	if cfg.SSHKeyPath == "" {
+		panic("SSH_KEY_PATH is empty or not set")
+	}
 	if cfg.EmailRecipient == "" {
 		panic("EMAIL_RECIPIENT is empty or not set")
 	}

pkg/upgrader/post-upgrade.go

@@ -0,0 +1,114 @@
+package upgrader
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+	yaml "gopkg.in/yaml.v3"
+
+	"git.csclub.uwaterloo.ca/cloud/cloudstack-k8s-upgrader/pkg/cloudstack"
+)
+
+var controlPlaneEndpointPattern = regexp.MustCompile("^https://([0-9.]+):6443/?$")
+
+func getControlPlaneIP(endpoint string) string {
+	matches := controlPlaneEndpointPattern.FindStringSubmatch(endpoint)
+	if matches == nil {
+		panic("Could not determine control plane IP from endpoint: " + endpoint)
+	}
+	return matches[1]
+}
+
+func (u *Upgrader) prepareSSHCommand(ipAddress string, args ...string) *exec.Cmd {
+	const user = "core"
+	log.Debug().
+		Str("user", user).
+		Str("address", ipAddress).
+		Msg("Running `" + strings.Join(args, " ") + "`")
+	args = append(
+		[]string{
+			"-i", u.cfg.SSHKeyPath,
+			"-o", "PasswordAuthentication=no",
+			"-o", "IdentitiesOnly=yes",
+			"-o", "StrictHostKeyChecking=no",
+			"-o", "CheckHostIP=no",
+			"-o", "UserKnownHostsFile=/dev/null",
+			"-o", "LogLevel=ERROR",
+			// -T is needed to pipe stdin
+			"-T",
+			user + "@" + ipAddress,
+		},
+		args...,
+	)
+	return exec.Command("ssh", args...)
+}
+
+const kubeControllerManagerConfigPath = "/etc/kubernetes/manifests/kube-controller-manager.yaml"
+const clusterSigningDurationArg = "--cluster-signing-duration=87600h"
+
+func interfaceSliceToStringSlice(arr1 []interface{}) []string {
+	arr2 := make([]string, len(arr1))
+	for i, elem := range arr1 {
+		arr2[i] = elem.(string)
+	}
+	return arr2
+}
+
+func (u *Upgrader) editKubeControllerManagerFlags(cluster *cloudstack.KubernetesClusterResponse) {
+	controlPlaneIP := getControlPlaneIP(cluster.Endpoint)
+	cmd := u.prepareSSHCommand(controlPlaneIP, "sudo", "cat", kubeControllerManagerConfigPath)
+	output, err := cmd.Output()
+	if err != nil {
+		panic(err)
+	}
+	manifest := make(map[string]interface{})
+	err = yaml.Unmarshal(output, &manifest)
+	if err != nil {
+		panic(err)
+	}
+	containers := manifest["spec"].(map[string]interface{})["containers"].([]interface{})
+	if len(containers) != 1 {
+		panic(fmt.Sprintf("Expected one command to be present, found %d", len(containers)))
+	}
+	container := containers[0].(map[string]interface{})
+	kubeControllerManagerArgv := interfaceSliceToStringSlice(container["command"].([]interface{}))
+	if kubeControllerManagerArgv[0] != "kube-controller-manager" {
+		panic(fmt.Sprintf("Expected argv[0] to be kube-controller-manager, got %s", kubeControllerManagerArgv[0]))
+	}
+	for _, arg := range kubeControllerManagerArgv[1:] {
+		if arg == clusterSigningDurationArg {
+			log.Debug().Msg("Found cluster signing duration arg, skipping")
+			return
+		}
+	}
+	// Insert the arg but keep argv[1:] sorted
+	newArgv := make([]string, 0, len(kubeControllerManagerArgv)+1)
+	newArgv = append(newArgv, kubeControllerManagerArgv[0])
+	insertedNewArg := false
+	for _, arg := range kubeControllerManagerArgv[1:] {
+		if !insertedNewArg && strings.Compare(arg, clusterSigningDurationArg) > 0 {
+			newArgv = append(newArgv, clusterSigningDurationArg)
+			insertedNewArg = true
+		}
+		newArgv = append(newArgv, arg)
+	}
+	if !insertedNewArg {
+		newArgv = append(newArgv, clusterSigningDurationArg)
+	}
+	container["command"] = newArgv
+	marshalledNewManifest, err := yaml.Marshal(&manifest)
+	if err != nil {
+		panic(err)
+	}
+	cmd = u.prepareSSHCommand(controlPlaneIP, "sudo", "tee", kubeControllerManagerConfigPath)
+	cmd.Stdin = bytes.NewReader(marshalledNewManifest)
+	cmd.Stdout = nil
+	err = cmd.Run()
+	if err != nil {
+		panic(err)
+	}
+}

pkg/upgrader/upgrader.go

@@ -185,6 +185,7 @@ func (u *Upgrader) DoUpgrade() {
 	if err != nil {
 		log.Warn().Msg("Could not delete old k8s ISO: " + err.Error())
 	}
+	u.editKubeControllerManagerFlags(cluster)
 	u.sendEmailNotification(nextVerStr)
 }