162 lines
3.9 KiB
Go
162 lines
3.9 KiB
Go
|
package cloudbuilder
|
||
|
|
||
|
import (
|
||
|
"bytes"
|
||
|
"encoding/json"
|
||
|
"fmt"
|
||
|
"net"
|
||
|
"os/exec"
|
||
|
"strings"
|
||
|
"time"
|
||
|
|
||
|
"github.com/rs/zerolog/log"
|
||
|
|
||
|
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config"
|
||
|
"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/distros"
|
||
|
)
|
||
|
|
||
|
type VMVerifier struct {
|
||
|
cfg *config.Config
|
||
|
user string
|
||
|
ipAddress string
|
||
|
hostname string
|
||
|
templateManager distros.ITemplateManager
|
||
|
}
|
||
|
|
||
|
func NewVMVerifier(
|
||
|
cfg *config.Config,
|
||
|
user string,
|
||
|
ipAddress string,
|
||
|
hostname string,
|
||
|
templateManager distros.ITemplateManager,
|
||
|
) *VMVerifier {
|
||
|
return &VMVerifier{
|
||
|
cfg: cfg,
|
||
|
user: user,
|
||
|
ipAddress: ipAddress,
|
||
|
hostname: hostname,
|
||
|
templateManager: templateManager,
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func waitUntilPort22IsOpen(ipAddress string) error {
|
||
|
const maxTimeToWait = 5 * time.Minute
|
||
|
const retryInterval = 5 * time.Second
|
||
|
const maxTries = int(maxTimeToWait / retryInterval)
|
||
|
address := ipAddress + ":22"
|
||
|
connected := false
|
||
|
for i := 0; i < maxTries; i++ {
|
||
|
_, err := net.DialTimeout("tcp", address, retryInterval)
|
||
|
if err == nil {
|
||
|
connected = true
|
||
|
break
|
||
|
}
|
||
|
if err.(*net.OpError).Timeout() {
|
||
|
log.Debug().Str("address", address).Msg("TCP connection timed out")
|
||
|
} else {
|
||
|
log.Debug().Str("address", address).Msg(err.Error())
|
||
|
time.Sleep(retryInterval)
|
||
|
}
|
||
|
}
|
||
|
if !connected {
|
||
|
return fmt.Errorf("Could not connect to %s", address)
|
||
|
}
|
||
|
return nil
|
||
|
}
|
||
|
|
||
|
func (v *VMVerifier) prepareSSHCommand(args ...string) *exec.Cmd {
|
||
|
log.Debug().
|
||
|
Str("user", v.user).
|
||
|
Str("address", v.ipAddress).
|
||
|
Msg("Running `" + strings.Join(args, " ") + "`")
|
||
|
args = append(
|
||
|
[]string{
|
||
|
"-i", v.cfg.SSHKeyPath,
|
||
|
"-o", "IdentitiesOnly=yes",
|
||
|
"-o", "StrictHostKeyChecking=no",
|
||
|
"-o", "CheckHostIP=no",
|
||
|
"-o", "UserKnownHostsFile=/dev/null",
|
||
|
"-o", "LogLevel=ERROR",
|
||
|
v.user + "@" + v.ipAddress,
|
||
|
},
|
||
|
args...,
|
||
|
)
|
||
|
return exec.Command("ssh", args...)
|
||
|
}
|
||
|
|
||
|
func (v *VMVerifier) runSSHCommand(args ...string) error {
|
||
|
return v.prepareSSHCommand(args...).Run()
|
||
|
}
|
||
|
|
||
|
func (v *VMVerifier) verifyThatVMCanResolveItsOwnHostname() error {
|
||
|
return v.runSSHCommand("ping", "-c", "1", "-w", "1", v.hostname)
|
||
|
}
|
||
|
|
||
|
func (v *VMVerifier) verifyThatSLAACandRAareDisabled() error {
|
||
|
output, err := v.prepareSSHCommand("ip", "-6", "addr", "show").Output()
|
||
|
if err != nil {
|
||
|
return err
|
||
|
}
|
||
|
if bytes.Contains(output, []byte("dynamic")) {
|
||
|
log.Debug().
|
||
|
Str("address", v.ipAddress).
|
||
|
Msg("IPv6 SLAAC address detected:\n" + string(output))
|
||
|
return fmt.Errorf("IPv6 SLAAC address detected")
|
||
|
}
|
||
|
output, err = v.prepareSSHCommand("ip", "-6", "route", "show").Output()
|
||
|
if err != nil {
|
||
|
return err
|
||
|
}
|
||
|
if bytes.Contains(output, []byte("proto ra")) {
|
||
|
log.Debug().
|
||
|
Str("address", v.ipAddress).
|
||
|
Msg("IPv6 RA route detected:\n" + string(output))
|
||
|
return fmt.Errorf("IPv6 RA route detected")
|
||
|
}
|
||
|
return nil
|
||
|
}
|
||
|
|
||
|
func (v *VMVerifier) verifyThatPackageCacheCanBeUpdated() error {
|
||
|
return v.runSSHCommand(v.templateManager.CommandToUpdatePackageCache()...)
|
||
|
}
|
||
|
|
||
|
func (v *VMVerifier) verifyThatCloudInitServicesDidNotFail() error {
|
||
|
output, err := v.prepareSSHCommand("systemctl", "list-units", "--state=failed", "-o", "json").Output()
|
||
|
if err != nil {
|
||
|
return err
|
||
|
}
|
||
|
var data []struct {
|
||
|
Unit string `json:"unit"`
|
||
|
Active string `json:"active"`
|
||
|
}
|
||
|
if err = json.Unmarshal(output, &data); err != nil {
|
||
|
return err
|
||
|
}
|
||
|
for _, unit := range data {
|
||
|
if strings.HasPrefix(unit.Unit, "cloud-") && unit.Active == "failed" {
|
||
|
return fmt.Errorf("unit %s failed", unit.Unit)
|
||
|
}
|
||
|
}
|
||
|
return nil
|
||
|
}
|
||
|
|
||
|
func (v *VMVerifier) Verify() (err error) {
|
||
|
log.Debug().Str("user", v.user).Str("address", v.ipAddress).Msg("Verifying VM")
|
||
|
if err = waitUntilPort22IsOpen(v.ipAddress); err != nil {
|
||
|
return
|
||
|
}
|
||
|
if err = v.verifyThatVMCanResolveItsOwnHostname(); err != nil {
|
||
|
return
|
||
|
}
|
||
|
if err = v.verifyThatSLAACandRAareDisabled(); err != nil {
|
||
|
return
|
||
|
}
|
||
|
if err = v.verifyThatPackageCacheCanBeUpdated(); err != nil {
|
||
|
return
|
||
|
}
|
||
|
if err = v.verifyThatCloudInitServicesDidNotFail(); err != nil {
|
||
|
return
|
||
|
}
|
||
|
return
|
||
|
}
|