cloudbuild/pkg/cloudbuilder/vm_verifier.go

package cloudbuilder

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net"
	"os/exec"
	"strings"
	"time"

	"github.com/rs/zerolog/log"

	"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/config"
	"git.csclub.uwaterloo.ca/cloud/cloudbuild/pkg/distros"
)

type VMVerifier struct {
	cfg             *config.Config
	user            string
	ipAddress       string
	hostname        string
	templateManager distros.ITemplateManager
}

func NewVMVerifier(
	cfg *config.Config,
	user string,
	ipAddress string,
	hostname string,
	templateManager distros.ITemplateManager,
) *VMVerifier {
	return &VMVerifier{
		cfg:             cfg,
		user:            user,
		ipAddress:       ipAddress,
		hostname:        hostname,
		templateManager: templateManager,
	}
}

func waitUntilPort22IsOpen(ipAddress string) error {
	const maxTimeToWait = 5 * time.Minute
	const retryInterval = 5 * time.Second
	const maxTries = int(maxTimeToWait / retryInterval)
	address := ipAddress + ":22"
	connected := false
	for i := 0; i < maxTries; i++ {
		_, err := net.DialTimeout("tcp", address, retryInterval)
		if err == nil {
			connected = true
			break
		}
		if err.(*net.OpError).Timeout() {
			log.Debug().Str("address", address).Msg("TCP connection timed out")
		} else {
			log.Debug().Str("address", address).Msg(err.Error())
			time.Sleep(retryInterval)
		}
	}
	if !connected {
		return fmt.Errorf("Could not connect to %s", address)
	}
	return nil
}

func (v *VMVerifier) prepareSSHCommand(args ...string) *exec.Cmd {
	log.Debug().
		Str("user", v.user).
		Str("address", v.ipAddress).
		Msg("Running `" + strings.Join(args, " ") + "`")
	args = append(
		[]string{
			"-i", v.cfg.SSHKeyPath,
			"-o", "IdentitiesOnly=yes",
			"-o", "StrictHostKeyChecking=no",
			"-o", "CheckHostIP=no",
			"-o", "UserKnownHostsFile=/dev/null",
			"-o", "LogLevel=ERROR",
			v.user + "@" + v.ipAddress,
		},
		args...,
	)
	return exec.Command("ssh", args...)
}

func (v *VMVerifier) runSSHCommand(args ...string) error {
	return v.prepareSSHCommand(args...).Run()
}

func (v *VMVerifier) verifyThatVMCanResolveItsOwnHostname() error {
	return v.runSSHCommand("ping", "-c", "1", "-w", "1", v.hostname)
}

func (v *VMVerifier) verifyThatSLAACandRAareDisabled() error {
	output, err := v.prepareSSHCommand("ip", "-6", "addr", "show").Output()
	if err != nil {
		return err
	}
	if bytes.Contains(output, []byte("dynamic")) {
		log.Debug().
			Str("address", v.ipAddress).
			Msg("IPv6 SLAAC address detected:\n" + string(output))
		return fmt.Errorf("IPv6 SLAAC address detected")
	}
	output, err = v.prepareSSHCommand("ip", "-6", "route", "show").Output()
	if err != nil {
		return err
	}
	if bytes.Contains(output, []byte("proto ra")) {
		log.Debug().
			Str("address", v.ipAddress).
			Msg("IPv6 RA route detected:\n" + string(output))
		return fmt.Errorf("IPv6 RA route detected")
	}
	return nil
}

func (v *VMVerifier) verifyThatPackageCacheCanBeUpdated() error {
	return v.runSSHCommand(v.templateManager.CommandToUpdatePackageCache()...)
}

func (v *VMVerifier) verifyThatCloudInitServicesDidNotFail() error {
	output, err := v.prepareSSHCommand("systemctl", "list-units", "--state=failed", "-o", "json").Output()
	if err != nil {
		return err
	}
	var data []struct {
		Unit   string `json:"unit"`
		Active string `json:"active"`
	}
	if err = json.Unmarshal(output, &data); err != nil {
		return err
	}
	for _, unit := range data {
		if strings.HasPrefix(unit.Unit, "cloud-") && unit.Active == "failed" {
			return fmt.Errorf("unit %s failed", unit.Unit)
		}
	}
	return nil
}

func (v *VMVerifier) Verify() (err error) {
	log.Debug().Str("user", v.user).Str("address", v.ipAddress).Msg("Verifying VM")
	if err = waitUntilPort22IsOpen(v.ipAddress); err != nil {
		return
	}
	if err = v.verifyThatVMCanResolveItsOwnHostname(); err != nil {
		return
	}
	if err = v.verifyThatSLAACandRAareDisabled(); err != nil {
		return
	}
	if err = v.verifyThatPackageCacheCanBeUpdated(); err != nil {
		return
	}
	if err = v.verifyThatCloudInitServicesDidNotFail(); err != nil {
		return
	}
	return
}