package cloudbuilder import ( "bytes" "encoding/json" "fmt" "net" "os/exec" "strings" "time" "github.com/rs/zerolog/log" "git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config" "git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/distros" ) type VMVerifier struct { cfg *config.Config user string ipAddress string hostname string templateManager distros.ITemplateManager } func NewVMVerifier( cfg *config.Config, user string, ipAddress string, hostname string, templateManager distros.ITemplateManager, ) *VMVerifier { return &VMVerifier{ cfg: cfg, user: user, ipAddress: ipAddress, hostname: hostname, templateManager: templateManager, } } func waitUntilPort22IsOpen(ipAddress string) error { const maxTimeToWait = 5 * time.Minute const retryInterval = 5 * time.Second const maxTries = int(maxTimeToWait / retryInterval) address := ipAddress + ":22" connected := false for i := 0; i < maxTries; i++ { _, err := net.DialTimeout("tcp", address, retryInterval) if err == nil { connected = true break } if err.(*net.OpError).Timeout() { log.Debug().Str("address", address).Msg("TCP connection timed out") } else { log.Debug().Str("address", address).Msg(err.Error()) time.Sleep(retryInterval) } } if !connected { return fmt.Errorf("Could not connect to %s", address) } return nil } func (v *VMVerifier) prepareSSHCommand(args ...string) *exec.Cmd { log.Debug(). Str("user", v.user). Str("address", v.ipAddress). Msg("Running `" + strings.Join(args, " ") + "`") args = append( []string{ "-i", v.cfg.SSHKeyPath, "-o", "IdentitiesOnly=yes", "-o", "StrictHostKeyChecking=no", "-o", "CheckHostIP=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR", v.user + "@" + v.ipAddress, }, args..., ) return exec.Command("ssh", args...) } func (v *VMVerifier) runSSHCommand(args ...string) error { return v.prepareSSHCommand(args...).Run() } func (v *VMVerifier) verifyThatVMCanResolveItsOwnHostname() error { return v.runSSHCommand("ping", "-c", "1", "-w", "1", v.hostname) } func (v *VMVerifier) verifyThatSLAACandRAareDisabled() error { output, err := v.prepareSSHCommand("ip", "-6", "addr", "show").Output() if err != nil { return err } if bytes.Contains(output, []byte("dynamic")) { log.Debug(). Str("address", v.ipAddress). Msg("IPv6 SLAAC address detected:\n" + string(output)) return fmt.Errorf("IPv6 SLAAC address detected") } output, err = v.prepareSSHCommand("ip", "-6", "route", "show").Output() if err != nil { return err } if bytes.Contains(output, []byte("proto ra")) { log.Debug(). Str("address", v.ipAddress). Msg("IPv6 RA route detected:\n" + string(output)) return fmt.Errorf("IPv6 RA route detected") } return nil } func (v *VMVerifier) verifyThatPackageCacheCanBeUpdated() error { return v.runSSHCommand(v.templateManager.CommandToUpdatePackageCache()...) } func (v *VMVerifier) verifyThatCloudInitServicesDidNotFail() error { output, err := v.prepareSSHCommand("systemctl", "list-units", "--state=failed", "-o", "json").Output() if err != nil { return err } var data []struct { Unit string `json:"unit"` Active string `json:"active"` } if err = json.Unmarshal(output, &data); err != nil { return err } for _, unit := range data { if strings.HasPrefix(unit.Unit, "cloud-") && unit.Active == "failed" { return fmt.Errorf("unit %s failed", unit.Unit) } } return nil } func (v *VMVerifier) Verify() (err error) { log.Debug().Str("user", v.user).Str("address", v.ipAddress).Msg("Verifying VM") if err = waitUntilPort22IsOpen(v.ipAddress); err != nil { return } if err = v.verifyThatVMCanResolveItsOwnHostname(); err != nil { return } if err = v.verifyThatSLAACandRAareDisabled(); err != nil { return } if err = v.verifyThatPackageCacheCanBeUpdated(); err != nil { return } if err = v.verifyThatCloudInitServicesDidNotFail(); err != nil { return } return }