parent
1f2428f804
commit
77d9fd3865
@ -0,0 +1,24 @@ |
||||
package cloudbuilder |
||||
|
||||
import "github.com/rs/zerolog/log" |
||||
|
||||
func (c *CloudBuilder) DeleteOldTemplates(oldTemplates map[string]TemplateVersionInfo, newVersions map[string]string) error { |
||||
for distro, _ := range newVersions { |
||||
oldTemplate, ok := oldTemplates[distro] |
||||
if !ok { |
||||
continue |
||||
} |
||||
log.Info(). |
||||
Str("oldVersion", oldTemplate.Version). |
||||
Str("distro", distro). |
||||
Msg("Deleting template") |
||||
jobID, err := c.client.DeleteTemplate(oldTemplate.TemplateId) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if err = c.client.WaitForJobToComplete(jobID); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,161 @@ |
||||
package cloudbuilder |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/json" |
||||
"fmt" |
||||
"net" |
||||
"os/exec" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/rs/zerolog/log" |
||||
|
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" |
||||
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/distros" |
||||
) |
||||
|
||||
type VMVerifier struct { |
||||
cfg *config.Config |
||||
user string |
||||
ipAddress string |
||||
hostname string |
||||
templateManager distros.ITemplateManager |
||||
} |
||||
|
||||
func NewVMVerifier( |
||||
cfg *config.Config, |
||||
user string, |
||||
ipAddress string, |
||||
hostname string, |
||||
templateManager distros.ITemplateManager, |
||||
) *VMVerifier { |
||||
return &VMVerifier{ |
||||
cfg: cfg, |
||||
user: user, |
||||
ipAddress: ipAddress, |
||||
hostname: hostname, |
||||
templateManager: templateManager, |
||||
} |
||||
} |
||||
|
||||
func waitUntilPort22IsOpen(ipAddress string) error { |
||||
const maxTimeToWait = 5 * time.Minute |
||||
const retryInterval = 5 * time.Second |
||||
const maxTries = int(maxTimeToWait / retryInterval) |
||||
address := ipAddress + ":22" |
||||
connected := false |
||||
for i := 0; i < maxTries; i++ { |
||||
_, err := net.DialTimeout("tcp", address, retryInterval) |
||||
if err == nil { |
||||
connected = true |
||||
break |
||||
} |
||||
if err.(*net.OpError).Timeout() { |
||||
log.Debug().Str("address", address).Msg("TCP connection timed out") |
||||
} else { |
||||
log.Debug().Str("address", address).Msg(err.Error()) |
||||
time.Sleep(retryInterval) |
||||
} |
||||
} |
||||
if !connected { |
||||
return fmt.Errorf("Could not connect to %s", address) |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (v *VMVerifier) prepareSSHCommand(args ...string) *exec.Cmd { |
||||
log.Debug(). |
||||
Str("user", v.user). |
||||
Str("address", v.ipAddress). |
||||
Msg("Running `" + strings.Join(args, " ") + "`") |
||||
args = append( |
||||
[]string{ |
||||
"-i", v.cfg.SSHKeyPath, |
||||
"-o", "IdentitiesOnly=yes", |
||||
"-o", "StrictHostKeyChecking=no", |
||||
"-o", "CheckHostIP=no", |
||||
"-o", "UserKnownHostsFile=/dev/null", |
||||
"-o", "LogLevel=ERROR", |
||||
v.user + "@" + v.ipAddress, |
||||
}, |
||||
args..., |
||||
) |
||||
return exec.Command("ssh", args...) |
||||
} |
||||
|
||||
func (v *VMVerifier) runSSHCommand(args ...string) error { |
||||
return v.prepareSSHCommand(args...).Run() |
||||
} |
||||
|
||||
func (v *VMVerifier) verifyThatVMCanResolveItsOwnHostname() error { |
||||
return v.runSSHCommand("ping", "-c", "1", "-w", "1", v.hostname) |
||||
} |
||||
|
||||
func (v *VMVerifier) verifyThatSLAACandRAareDisabled() error { |
||||
output, err := v.prepareSSHCommand("ip", "-6", "addr", "show").Output() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if bytes.Contains(output, []byte("dynamic")) { |
||||
log.Debug(). |
||||
Str("address", v.ipAddress). |
||||
Msg("IPv6 SLAAC address detected:\n" + string(output)) |
||||
return fmt.Errorf("IPv6 SLAAC address detected") |
||||
} |
||||
output, err = v.prepareSSHCommand("ip", "-6", "route", "show").Output() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if bytes.Contains(output, []byte("proto ra")) { |
||||
log.Debug(). |
||||
Str("address", v.ipAddress). |
||||
Msg("IPv6 RA route detected:\n" + string(output)) |
||||
return fmt.Errorf("IPv6 RA route detected") |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (v *VMVerifier) verifyThatPackageCacheCanBeUpdated() error { |
||||
return v.runSSHCommand(v.templateManager.CommandToUpdatePackageCache()...) |
||||
} |
||||
|
||||
func (v *VMVerifier) verifyThatCloudInitServicesDidNotFail() error { |
||||
output, err := v.prepareSSHCommand("systemctl", "list-units", "--state=failed", "-o", "json").Output() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
var data []struct { |
||||
Unit string `json:"unit"` |
||||
Active string `json:"active"` |
||||
} |
||||
if err = json.Unmarshal(output, &data); err != nil { |
||||
return err |
||||
} |
||||
for _, unit := range data { |
||||
if strings.HasPrefix(unit.Unit, "cloud-") && unit.Active == "failed" { |
||||
return fmt.Errorf("unit %s failed", unit.Unit) |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (v *VMVerifier) Verify() (err error) { |
||||
log.Debug().Str("user", v.user).Str("address", v.ipAddress).Msg("Verifying VM") |
||||
if err = waitUntilPort22IsOpen(v.ipAddress); err != nil { |
||||
return |
||||
} |
||||
if err = v.verifyThatVMCanResolveItsOwnHostname(); err != nil { |
||||
return |
||||
} |
||||
if err = v.verifyThatSLAACandRAareDisabled(); err != nil { |
||||
return |
||||
} |
||||
if err = v.verifyThatPackageCacheCanBeUpdated(); err != nil { |
||||
return |
||||
} |
||||
if err = v.verifyThatCloudInitServicesDidNotFail(); err != nil { |
||||
return |
||||
} |
||||
return |
||||
} |
||||
Loading…
Reference in new issue