mirror/merlin/sync/command.go

package sync

import (
	"fmt"
	"math/rand"
	"strconv"

	"git.csclub.uwaterloo.ca/public/merlin/config"
)

func buildRsyncHost(repo *config.Repo) string {
	if repo.RsyncUser != "" {
		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
	}
	return "rsync://" + repo.RsyncHost + "/" + repo.RsyncDir
}

func buildRsyncDaemonHost(repo *config.Repo) string {
	if repo.RsyncUser != "" {
		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
	}
	return repo.RsyncHost + "::" + repo.RsyncDir
}

func buildRsyncSSHHost(repo *config.Repo) string {
	if repo.RsyncUser != "" {
		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
	}
	return repo.RsyncHost + ":" + repo.RsyncDir
}

func buildDownloadDir(repo *config.Repo) string {
	return config.Conf.DownloadDir + "/" + repo.LocalDir
}

func cscSyncApache(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-az", "--no-owner", "--no-group", "--delete", "--safe-links",
		"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/",
		"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncArchLinux(repo *config.Repo) []string {

	tempDir := "" // is this option even needed?

	// add option for verbose flag?
	args := []string{
		"rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600",
		"--contimeout=60", "-p", "--delay-updates", "--no-motd",
		"--temp-dir=" + tempDir, "--log-file=" + repo.RsyncLogFile, "--address=" + config.Conf.IPv4Address,
	}
	args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))

	return args

}

func cscSyncBadPerms(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--chmod=o=rX", "--delete",
		"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/",
		"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))

	return args
}

// TODO ceph

func cscSyncCDImage(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete",
		"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".*/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncChmod(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", "--delay-updates", "--safe-links",
		"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
		"--chmod=u=rwX,go=rX",
	}
	args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncDebian(repo *config.Repo) []string {

	// sync /pool
	args := []string{"nice", "rsync", "-rlHtvp",
		"--exclude", ".~tmp~/", "--timeout=3600", "-4",
		"--address=" + config.Conf.IPv4Address,
	}
	// $RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1
	return args
}

func cscSyncDebianCD(repo *config.Repo) []string {

	// this is basically the same as CSCSyncDebian, except it has an extra --exclude
	args := []string{"nice", "rsync", "-rlHtvp", "--delete",
		"--exclude", ".~tmp~/", "--timeout=3600", "-4",
		"--address=" + config.Conf.IPv4Address,
		// "--exclude", "Archive-Update-in-Progress-${HOSTNAME}"
	}
	// $RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
	return args
}

func cscSyncGentoo(repo *config.Repo) []string {
	repo.RsyncUser = "gentoo"
	repo.PasswordFile = "gentoo-distfiles"
	return cscSyncStandard(repo)
}

// TODO s3

func cscSyncSSH(repo *config.Repo) []string {

	args := []string{
		"rsync", "-aH", "--no-owner", "--no-group", "--delete",
		"--timeout=3600", "-4",
		"--exclude", ".~tmp~/",
		"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	// not sure if we should be assuming ssh identity file is the password file
	args = append(args, "-e", fmt.Sprintf("'ssh -i %s'", repo.PasswordFile))
	args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncStandard(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
		"--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	if repo.PasswordFile != "" {
		filename := config.Conf.PasswordDir + "/" + repo.PasswordFile
		args = append(args, "--password-file", filename)
	}
	args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncStandardIPV6(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
		"--delay-updates", "--safe-links", "--timeout=3600", "-6", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))

	return args
}

// for testing, to be removed later
func cscSyncDummy(repo *config.Repo) []string {

	sleepDur := strconv.FormatInt(rand.Int63n(10)+5, 10)
	args := []string{"sleep", sleepDur}

	return args
}

// executes a particular sync job depending on repo.SyncType.
func getSyncCommand(repo *config.Repo) []string {
	/*
		# scripts used by merlin.py
		csc-sync-debian
		csc-sync-standard
		csc-sync-ssh
		csc-sync-apache
		csc-sync-archlinux
		csc-sync-debian-cd
		csc-sync-gentoo
		csc-sync-wget
		csc-sync-s3
		csc-sync-standard-ipv6
		csc-sync-ceph

		zfssync
		report_mirror (what is this?)

		# other things in bin/
		csc-sync-archlinux-old
		csc-sync-cdimage
		csc-sync-chmod
		csc-sync-badperms
		make-torrents
		ubuntu-releases-sync
	*/
	switch repo.SyncType {

	case "csc-sync-apache":
		return cscSyncApache(repo)
	case "csc-sync-archlinux":
		return cscSyncArchLinux(repo)
	case "csc-sync-badperms":
		return cscSyncBadPerms(repo)
	case "csc-sync-cdimage":
		return cscSyncCDImage(repo)
	// case "csc-sync-ceph":
	// 	return cscSyncCeph(repo)
	case "csc-sync-chmod":
		return cscSyncChmod(repo)
	case "csc-sync-debian":
		return cscSyncDebian(repo)
	case "csc-sync-debian-cd":
		return cscSyncDebianCD(repo)
	case "csc-sync-gentoo":
		return cscSyncGentoo(repo)
	// case "csc-sync-s3":
	// 	return cscSyncS3(repo)
	case "csc-sync-ssh":
		return cscSyncSSH(repo)
	case "csc-sync-standard":
		return cscSyncStandard(repo)
	case "csc-sync-standard-ipv6":
		return cscSyncStandardIPV6(repo)
		// case "csc-sync-wget":
		// 	return cscSyncWget(repo)
	case "csc-sync-dummy":
		return cscSyncDummy(repo)
	default:
		repo.Logger.Error("Unrecognized sync type", "'"+repo.SyncType+"'")
	}

	return []string{}
}